From 9750cfb2fad349ef4a9a57f7086d9103a1a82e18 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 11 Feb 2026 00:00:32 -0500 Subject: [PATCH 1/6] add cached count fields to avoid expensive COUNT queries - Add sec_updates_count, bug_updates_count, packages_count, errata_count to Host - Add packages_count to Mirror - Add hosts_count to OSVariant - Add Django signals to auto-update counts on M2M/FK changes - Update views to use cached fields instead of annotations --- hosts/apps.py | 3 + ...pdates_count_host_errata_count_and_more.py | 33 +++++++++++ .../migrations/0012_backfill_cached_counts.py | 33 +++++++++++ hosts/models.py | 16 ++++-- hosts/serializers.py | 4 +- hosts/signals.py | 45 +++++++++++++++ hosts/tables.py | 2 +- hosts/tasks.py | 8 +-- hosts/views.py | 10 +--- operatingsystems/apps.py | 3 + .../migrations/0009_osvariant_hosts_count.py | 18 ++++++ .../0010_backfill_osvariant_counts.py | 29 ++++++++++ operatingsystems/models.py | 2 + operatingsystems/signals.py | 56 +++++++++++++++++++ operatingsystems/views.py | 2 +- repos/apps.py | 3 + .../migrations/0008_mirror_packages_count.py | 18 ++++++ .../migrations/0009_backfill_mirror_counts.py | 27 +++++++++ repos/models.py | 2 + repos/signals.py | 28 ++++++++++ repos/tables.py | 2 +- repos/views.py | 7 +-- sbin/patchman | 6 +- util/views.py | 19 ++++--- 24 files changed, 339 insertions(+), 37 deletions(-) create mode 100644 hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py create mode 100644 hosts/migrations/0012_backfill_cached_counts.py create mode 100644 hosts/signals.py create mode 100644 operatingsystems/migrations/0009_osvariant_hosts_count.py create mode 100644 operatingsystems/migrations/0010_backfill_osvariant_counts.py create mode 100644 operatingsystems/signals.py create mode 100644 repos/migrations/0008_mirror_packages_count.py create mode 100644 repos/migrations/0009_backfill_mirror_counts.py create mode 100644 repos/signals.py diff --git a/hosts/apps.py b/hosts/apps.py index 84efc94f..a805f23e 100644 --- a/hosts/apps.py +++ b/hosts/apps.py @@ -19,3 +19,6 @@ class HostsConfig(AppConfig): name = 'hosts' + + def ready(self): + import hosts.signals # noqa diff --git a/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py b/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py new file mode 100644 index 00000000..58e363d0 --- /dev/null +++ b/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.28 on 2026-02-11 04:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0010_alter_hostrepo_options'), + ] + + operations = [ + migrations.AddField( + model_name='host', + name='bug_updates_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + migrations.AddField( + model_name='host', + name='errata_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + migrations.AddField( + model_name='host', + name='packages_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + migrations.AddField( + model_name='host', + name='sec_updates_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + ] diff --git a/hosts/migrations/0012_backfill_cached_counts.py b/hosts/migrations/0012_backfill_cached_counts.py new file mode 100644 index 00000000..4e7b515d --- /dev/null +++ b/hosts/migrations/0012_backfill_cached_counts.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.28 on 2026-02-11 + +from django.db import migrations + + +def backfill_host_counts(apps, schema_editor): + """Backfill cached count fields for existing hosts.""" + Host = apps.get_model('hosts', 'Host') + for host in Host.objects.all(): + host.sec_updates_count = host.updates.filter(security=True).count() + host.bug_updates_count = host.updates.filter(security=False).count() + host.packages_count = host.packages.count() + host.errata_count = host.errata.count() + host.save(update_fields=[ + 'sec_updates_count', 'bug_updates_count', + 'packages_count', 'errata_count' + ]) + + +def reverse_backfill(apps, schema_editor): + """No-op reverse - counts will be recalculated on next report.""" + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0011_host_bug_updates_count_host_errata_count_and_more'), + ] + + operations = [ + migrations.RunPython(backfill_host_counts, reverse_backfill), + ] diff --git a/hosts/models.py b/hosts/models.py index 7c646b98..fc8b7fa0 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -60,6 +60,11 @@ class Host(models.Model): tags = TaggableManager(blank=True) updated_at = models.DateTimeField(default=timezone.now) errata = models.ManyToManyField(Erratum, blank=True) + # Cached count fields for query optimization + sec_updates_count = models.PositiveIntegerField(default=0, db_index=True) + bug_updates_count = models.PositiveIntegerField(default=0, db_index=True) + packages_count = models.PositiveIntegerField(default=0, db_index=True) + errata_count = models.PositiveIntegerField(default=0, db_index=True) from hosts.managers import HostManager objects = HostManager() @@ -97,20 +102,23 @@ def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) def get_num_security_updates(self): - return self.updates.filter(security=True).count() + return self.sec_updates_count def get_num_bugfix_updates(self): - return self.updates.filter(security=False).count() + return self.bug_updates_count def get_num_updates(self): - return self.updates.count() + return self.sec_updates_count + self.bug_updates_count def get_num_packages(self): - return self.packages.count() + return self.packages_count def get_num_repos(self): return self.repos.count() + def get_num_errata(self): + return self.errata_count + def check_rdns(self): if self.check_dns: update_rdns(self) diff --git a/hosts/serializers.py b/hosts/serializers.py index c0313e47..9565bae6 100644 --- a/hosts/serializers.py +++ b/hosts/serializers.py @@ -33,10 +33,10 @@ class Meta: 'updated_at', 'bugfix_update_count', 'security_update_count') def get_bugfix_update_count(self, obj): - return obj.updates.filter(security=False).count() + return obj.bug_updates_count def get_security_update_count(self, obj): - return obj.updates.filter(security=True).count() + return obj.sec_updates_count class HostRepoSerializer(serializers.HyperlinkedModelSerializer): diff --git a/hosts/signals.py b/hosts/signals.py new file mode 100644 index 00000000..ca5de922 --- /dev/null +++ b/hosts/signals.py @@ -0,0 +1,45 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.db.models.signals import m2m_changed +from django.dispatch import receiver + +from hosts.models import Host + + +@receiver(m2m_changed, sender=Host.packages.through) +def update_host_packages_count(sender, instance, action, **kwargs): + """Update packages_count when Host.packages M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.packages_count = instance.packages.count() + instance.save(update_fields=['packages_count']) + + +@receiver(m2m_changed, sender=Host.updates.through) +def update_host_updates_count(sender, instance, action, **kwargs): + """Update sec_updates_count and bug_updates_count when Host.updates M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.sec_updates_count = instance.updates.filter(security=True).count() + instance.bug_updates_count = instance.updates.filter(security=False).count() + instance.save(update_fields=['sec_updates_count', 'bug_updates_count']) + + +@receiver(m2m_changed, sender=Host.errata.through) +def update_host_errata_count(sender, instance, action, **kwargs): + """Update errata_count when Host.errata M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.errata_count = instance.errata.count() + instance.save(update_fields=['errata_count']) diff --git a/hosts/tables.py b/hosts/tables.py index 715a29ae..b34b7d61 100644 --- a/hosts/tables.py +++ b/hosts/tables.py @@ -31,7 +31,7 @@ '{% endwith %}' ) AFFECTED_ERRATA_TEMPLATE = ( - '{% with count=record.errata.count %}' + '{% with count=record.errata_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' diff --git a/hosts/tasks.py b/hosts/tasks.py index f186760f..a8632a5d 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task -from django.db.models import Count from hosts.models import Host from util import get_datetime_now @@ -51,10 +50,9 @@ def find_all_host_updates_homogenous(): host.save() # only include hosts with the exact same number of packages - filtered_hosts = Host.objects.annotate( - packages_count=Count('packages')).filter( - packages_count=host.packages.count() - ) + filtered_hosts = Host.objects.filter( + packages_count=host.packages_count + ) # and exclude hosts with the current timestamp filtered_hosts = filtered_hosts.exclude(updated_at=ts) diff --git a/hosts/views.py b/hosts/views.py index 259cef57..ae63665c 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -17,7 +17,7 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.db.models import Count, Q +from django.db.models import Q from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django_filters import rest_framework as filters @@ -76,12 +76,8 @@ def _get_filtered_hosts(filter_params): @login_required def host_list(request): - hosts = Host.objects.select_related().annotate( - sec_updates_count=Count('updates', filter=Q(updates__security=True), distinct=True), - bug_updates_count=Count('updates', filter=Q(updates__security=False), distinct=True), - errata_count=Count('errata', distinct=True), - packages_count=Count('packages', distinct=True), - ) + # Use cached count fields instead of expensive annotations + hosts = Host.objects.select_related() if 'domain_id' in request.GET: hosts = hosts.filter(domain=request.GET['domain_id']) diff --git a/operatingsystems/apps.py b/operatingsystems/apps.py index 4a7f65f7..b0467e42 100644 --- a/operatingsystems/apps.py +++ b/operatingsystems/apps.py @@ -19,3 +19,6 @@ class OperatingsystemsConfig(AppConfig): name = 'operatingsystems' + + def ready(self): + import operatingsystems.signals # noqa diff --git a/operatingsystems/migrations/0009_osvariant_hosts_count.py b/operatingsystems/migrations/0009_osvariant_hosts_count.py new file mode 100644 index 00000000..e388f1ad --- /dev/null +++ b/operatingsystems/migrations/0009_osvariant_hosts_count.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-02-11 04:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('operatingsystems', '0008_alter_osrelease_options_alter_osvariant_options'), + ] + + operations = [ + migrations.AddField( + model_name='osvariant', + name='hosts_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + ] diff --git a/operatingsystems/migrations/0010_backfill_osvariant_counts.py b/operatingsystems/migrations/0010_backfill_osvariant_counts.py new file mode 100644 index 00000000..a14fc90d --- /dev/null +++ b/operatingsystems/migrations/0010_backfill_osvariant_counts.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.28 on 2026-02-11 + +from django.db import migrations + + +def backfill_osvariant_counts(apps, schema_editor): + """Backfill hosts_count for existing osvariants.""" + OSVariant = apps.get_model('operatingsystems', 'OSVariant') + Host = apps.get_model('hosts', 'Host') + for osvariant in OSVariant.objects.all(): + osvariant.hosts_count = Host.objects.filter(osvariant=osvariant).count() + osvariant.save(update_fields=['hosts_count']) + + +def reverse_backfill(apps, schema_editor): + """No-op reverse.""" + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('operatingsystems', '0009_osvariant_hosts_count'), + ('hosts', '0012_backfill_cached_counts'), + ] + + operations = [ + migrations.RunPython(backfill_osvariant_counts, reverse_backfill), + ] diff --git a/operatingsystems/models.py b/operatingsystems/models.py index 234b8ab3..911332a9 100644 --- a/operatingsystems/models.py +++ b/operatingsystems/models.py @@ -57,6 +57,8 @@ class OSVariant(models.Model): arch = models.ForeignKey(MachineArchitecture, blank=True, null=True, on_delete=models.CASCADE) osrelease = models.ForeignKey(OSRelease, blank=True, null=True, on_delete=models.SET_NULL) codename = models.CharField(max_length=255, blank=True) + # Cached count field for query optimization + hosts_count = models.PositiveIntegerField(default=0, db_index=True) class Meta: verbose_name = 'Operating System Variant' diff --git a/operatingsystems/signals.py b/operatingsystems/signals.py new file mode 100644 index 00000000..5f1d0507 --- /dev/null +++ b/operatingsystems/signals.py @@ -0,0 +1,56 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.db.models.signals import post_delete, post_save, pre_save +from django.dispatch import receiver + +from hosts.models import Host + + +@receiver(pre_save, sender=Host) +def track_osvariant_change(sender, instance, **kwargs): + """Track old osvariant before save to update its count.""" + if instance.pk: + try: + old_instance = Host.objects.get(pk=instance.pk) + instance._old_osvariant = old_instance.osvariant + except Host.DoesNotExist: + instance._old_osvariant = None + else: + instance._old_osvariant = None + + +@receiver(post_save, sender=Host) +def update_osvariant_count_on_save(sender, instance, created, **kwargs): + """Update OSVariant.hosts_count when Host is created or osvariant changes.""" + # Update new osvariant count + if instance.osvariant: + instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count() + instance.osvariant.save(update_fields=['hosts_count']) + + # Update old osvariant count if it changed + old_osvariant = getattr(instance, '_old_osvariant', None) + if old_osvariant and old_osvariant != instance.osvariant: + old_osvariant.hosts_count = Host.objects.filter(osvariant=old_osvariant).count() + old_osvariant.save(update_fields=['hosts_count']) + + +@receiver(post_delete, sender=Host) +def update_osvariant_count_on_delete(sender, instance, **kwargs): + """Update OSVariant.hosts_count when Host is deleted.""" + if instance.osvariant: + instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count() + instance.osvariant.save(update_fields=['hosts_count']) diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 7dd4fcb5..91b273aa 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -77,8 +77,8 @@ def _get_filtered_osreleases(filter_params): @login_required def osvariant_list(request): + # Use cached hosts_count instead of expensive annotation osvariants = OSVariant.objects.select_related().annotate( - hosts_count=Count('host'), repos_count=Count('osrelease__repos'), ) diff --git a/repos/apps.py b/repos/apps.py index 4f62987a..d0cf38ed 100644 --- a/repos/apps.py +++ b/repos/apps.py @@ -19,3 +19,6 @@ class ReposConfig(AppConfig): name = 'repos' + + def ready(self): + import repos.signals # noqa diff --git a/repos/migrations/0008_mirror_packages_count.py b/repos/migrations/0008_mirror_packages_count.py new file mode 100644 index 00000000..41c99fe8 --- /dev/null +++ b/repos/migrations/0008_mirror_packages_count.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-02-11 04:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('repos', '0007_alter_mirror_options_alter_mirrorpackage_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='mirror', + name='packages_count', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + ] diff --git a/repos/migrations/0009_backfill_mirror_counts.py b/repos/migrations/0009_backfill_mirror_counts.py new file mode 100644 index 00000000..244bf64b --- /dev/null +++ b/repos/migrations/0009_backfill_mirror_counts.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.28 on 2026-02-11 + +from django.db import migrations + + +def backfill_mirror_counts(apps, schema_editor): + """Backfill packages_count for existing mirrors.""" + Mirror = apps.get_model('repos', 'Mirror') + for mirror in Mirror.objects.all(): + mirror.packages_count = mirror.packages.count() + mirror.save(update_fields=['packages_count']) + + +def reverse_backfill(apps, schema_editor): + """No-op reverse.""" + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('repos', '0008_mirror_packages_count'), + ] + + operations = [ + migrations.RunPython(backfill_mirror_counts, reverse_backfill), + ] diff --git a/repos/models.py b/repos/models.py index 8af9377a..421bca00 100644 --- a/repos/models.py +++ b/repos/models.py @@ -151,6 +151,8 @@ class Mirror(models.Model): enabled = models.BooleanField(default=True) refresh = models.BooleanField(default=True) fail_count = models.IntegerField(default=0) + # Cached count field for query optimization + packages_count = models.PositiveIntegerField(default=0, db_index=True) class Meta: verbose_name_plural = 'Mirror' diff --git a/repos/signals.py b/repos/signals.py new file mode 100644 index 00000000..aa014736 --- /dev/null +++ b/repos/signals.py @@ -0,0 +1,28 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.db.models.signals import m2m_changed +from django.dispatch import receiver + +from repos.models import Mirror + + +@receiver(m2m_changed, sender=Mirror.packages.through) +def update_mirror_packages_count(sender, instance, action, **kwargs): + """Update packages_count when Mirror.packages M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.packages_count = instance.packages.count() + instance.save(update_fields=['packages_count']) diff --git a/repos/tables.py b/repos/tables.py index cff76c08..a0e3aca0 100644 --- a/repos/tables.py +++ b/repos/tables.py @@ -36,7 +36,7 @@ MIRROR_PACKAGES_TEMPLATE = ( '{% if not record.mirrorlist %}' '' - '{{ record.packages.count }}{% endif %}' + '{{ record.packages_count }}{% endif %}' ) MIRROR_ENABLED_TEMPLATE = '{% load common %}{% yes_no_img record.enabled %}' REFRESH_TEMPLATE = '{% load common %}{% yes_no_img record.refresh %}' diff --git a/repos/views.py b/repos/views.py index 2a45b668..58a9cdd9 100644 --- a/repos/views.py +++ b/repos/views.py @@ -18,7 +18,7 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db import IntegrityError -from django.db.models import Count, Q +from django.db.models import Q from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse @@ -149,9 +149,8 @@ def move_mirrors(repo): if oldrepo.mirror_set.count() == 0: oldrepo.delete() - mirrors = Mirror.objects.select_related().annotate( - packages_count=Count('packages'), - ).order_by('packages_checksum') + # Use cached packages_count instead of expensive annotation + mirrors = Mirror.objects.select_related().order_by('packages_checksum') checksum = None if 'checksum' in request.GET: diff --git a/sbin/patchman b/sbin/patchman index 06bef981..5dabdfa7 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -23,7 +23,6 @@ import sys from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned -from django.db.models import Count os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa @@ -174,9 +173,8 @@ def host_updates_alt(host=None): host.save() # only include hosts with the same number of packages - filtered_hosts = Host.objects.annotate( - packages_count=Count('packages')).filter( - packages_count=host.packages.count()) + filtered_hosts = Host.objects.filter( + packages_count=host.packages_count) # exclude hosts with the current timestamp filtered_hosts = filtered_hosts.exclude(updated_at=ts) diff --git a/util/views.py b/util/views.py index 6d88f28d..073188a1 100644 --- a/util/views.py +++ b/util/views.py @@ -55,8 +55,9 @@ def dashboard(request): stale_hosts = hosts.filter(lastreport__lt=last_report_delta) norepo_hosts = hosts.filter(repos__isnull=True, osvariant__osrelease__repos__isnull=True) # noqa reboot_hosts = hosts.filter(reboot_required=True) - secupdate_hosts = hosts.filter(updates__security=True, updates__isnull=False).distinct() # noqa - bugupdate_hosts = hosts.exclude(updates__security=True, updates__isnull=False).distinct().filter(updates__security=False, updates__isnull=False).distinct() # noqa + # Use cached count fields instead of expensive M2M JOINs + secupdate_hosts = hosts.filter(sec_updates_count__gt=0) + bugupdate_hosts = hosts.filter(bug_updates_count__gt=0, sec_updates_count=0) diff_rdns_hosts = hosts.exclude(reversedns=F('hostname')).filter(check_dns=True) # noqa # os variant issues @@ -89,14 +90,16 @@ def dashboard(request): checksums = {} possible_mirrors = {} - for csvalue in Mirror.objects.all().values('packages_checksum').distinct(): + # Use cached packages_count to avoid N+1 queries + for csvalue in Mirror.objects.filter(packages_count__gt=0).values('packages_checksum').distinct(): checksum = csvalue['packages_checksum'] if checksum is not None and checksum != 'yast': - for mirror in Mirror.objects.filter(packages_checksum=checksum): - if mirror.packages.count() > 0: - if checksum not in checksums: - checksums[checksum] = [] - checksums[checksum].append(mirror) + mirrors = list(Mirror.objects.filter( + packages_checksum=checksum, + packages_count__gt=0 + ).select_related('repo')) + if mirrors: + checksums[checksum] = mirrors for checksum in checksums: first_mirror = checksums[checksum][0] From 3f4cb692f113284d75d9c896eaf7dad43fe756d9 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 11 Feb 2026 21:08:25 -0500 Subject: [PATCH 2/6] add further sql optimizations --- errata/managers.py | 2 +- errata/views.py | 2 +- hosts/managers.py | 2 +- hosts/models.py | 12 +++++----- hosts/views.py | 10 ++++---- modules/managers.py | 2 +- modules/views.py | 4 ++-- operatingsystems/forms.py | 2 +- .../operatingsystems/osvariant_detail.html | 2 +- operatingsystems/views.py | 10 ++++---- packages/managers.py | 2 +- packages/utils.py | 5 ++-- packages/views.py | 10 ++++---- reports/views.py | 4 ++-- repos/forms.py | 4 ++-- repos/managers.py | 2 +- repos/models.py | 24 +++++++------------ repos/utils.py | 5 ++-- repos/views.py | 20 +++++++--------- security/managers.py | 2 +- security/views.py | 6 ++--- util/management/commands/revoke_api_key.py | 2 +- util/templatetags/common.py | 6 ++--- 23 files changed, 65 insertions(+), 75 deletions(-) diff --git a/errata/managers.py b/errata/managers.py index e39147be..b8ab16a8 100644 --- a/errata/managers.py +++ b/errata/managers.py @@ -19,4 +19,4 @@ class ErratumManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset() diff --git a/errata/views.py b/errata/views.py index 285f7483..21382eff 100644 --- a/errata/views.py +++ b/errata/views.py @@ -29,7 +29,7 @@ @login_required def erratum_list(request): - errata = Erratum.objects.select_related() + errata = Erratum.objects.all() if 'e_type' in request.GET: errata = errata.filter(e_type=request.GET['e_type']).distinct() diff --git a/hosts/managers.py b/hosts/managers.py index 85a489d4..73301b58 100644 --- a/hosts/managers.py +++ b/hosts/managers.py @@ -20,4 +20,4 @@ class HostManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('osvariant', 'arch', 'domain') diff --git a/hosts/models.py b/hosts/models.py index fc8b7fa0..96f163bf 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -135,9 +135,9 @@ def clean_reports(self): """ Remove all but the last 3 reports for a host """ from reports.models import Report - reports = Report.objects.filter(host=self).order_by('-created')[3:] - rlen = reports.count() - for report in Report.objects.filter(host=self).order_by('-created')[3:]: + reports = list(Report.objects.filter(host=self).order_by('-created')[3:]) + rlen = len(reports) + for report in reports: report.delete() if rlen > 0: info_message(text=f'{self.hostname}: removed {rlen} old reports') @@ -157,14 +157,14 @@ def get_host_repo_packages(self): Q(mirror__repo__in=self.repos.all(), mirror__enabled=True, mirror__repo__enabled=True) - return Package.objects.select_related().filter(hostrepos_q).distinct() + return Package.objects.select_related('name', 'arch').filter(hostrepos_q).distinct() def process_update(self, package, highest_package): if self.host_repos_only: host_repos = Q(repo__host=self) else: host_repos = Q(repo__osrelease__osvariant__host=self, repo__arch=self.arch) | Q(repo__host=self) - mirrors = highest_package.mirror_set.filter(host_repos) + mirrors = highest_package.mirror_set.filter(host_repos).select_related('repo') security = False # if any of the containing repos are security, mark the update as a security update for mirror in mirrors: @@ -223,7 +223,7 @@ def find_host_repo_updates(self, host_packages, repo_packages): repo__mirror__refresh=True, repo__mirror__repo__enabled=True, host=self) - hostrepos = HostRepo.objects.select_related().filter(hostrepos_q) + hostrepos = HostRepo.objects.select_related('host', 'repo').filter(hostrepos_q) for package in host_packages: highest_package = package diff --git a/hosts/views.py b/hosts/views.py index ae63665c..f905f8c6 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -42,7 +42,7 @@ def _get_filtered_hosts(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - hosts = Host.objects.select_related() + hosts = Host.objects.select_related('osvariant', 'arch', 'domain') if 'domain_id' in params: hosts = hosts.filter(domain=params['domain_id'][0]) @@ -77,7 +77,7 @@ def _get_filtered_hosts(filter_params): @login_required def host_list(request): # Use cached count fields instead of expensive annotations - hosts = Host.objects.select_related() + hosts = Host.objects.select_related('osvariant', 'arch', 'domain') if 'domain_id' in request.GET: hosts = hosts.filter(domain=request.GET['domain_id']) @@ -156,7 +156,7 @@ def host_detail(request, hostname): hostrepos = HostRepo.objects.filter(host=host) # Build packages list with update info - updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related()} + updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related('oldpackage', 'newpackage')} packages_with_updates = [] for package in host.packages.select_related('name', 'arch').order_by('name__name'): package.update = updates_by_package.get(package.id) @@ -294,7 +294,7 @@ class HostViewSet(viewsets.ModelViewSet): """ API endpoint that allows hosts to be viewed or edited. """ - queryset = Host.objects.all() + queryset = Host.objects.select_related('osvariant', 'arch', 'domain').all() serializer_class = HostSerializer filterset_class = HostFilter @@ -303,5 +303,5 @@ class HostRepoViewSet(viewsets.ModelViewSet): """ API endpoint that allows host repos to be viewed or edited. """ - queryset = HostRepo.objects.all() + queryset = HostRepo.objects.select_related('host', 'repo').all() serializer_class = HostRepoSerializer diff --git a/modules/managers.py b/modules/managers.py index ebf7fe31..ca0a6dee 100644 --- a/modules/managers.py +++ b/modules/managers.py @@ -19,4 +19,4 @@ class ModuleManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('arch', 'repo') diff --git a/modules/views.py b/modules/views.py index 45703f29..bde0b741 100644 --- a/modules/views.py +++ b/modules/views.py @@ -28,7 +28,7 @@ @login_required def module_list(request): - modules = Module.objects.select_related() + modules = Module.objects.select_related('arch', 'repo') if 'search' in request.GET: terms = request.GET['search'].lower() @@ -62,6 +62,6 @@ class ModuleViewSet(viewsets.ModelViewSet): """ API endpoint that allows modules to be viewed or edited. """ - queryset = Module.objects.all() + queryset = Module.objects.select_related('arch', 'repo').all() serializer_class = ModuleSerializer permission_classes = (permissions.IsAuthenticatedOrReadOnly,) diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index fa319182..5d063b42 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -47,7 +47,7 @@ class Meta: class AddReposToOSReleaseForm(ModelForm): repos = ModelMultipleChoiceField( - queryset=Repository.objects.select_related(), + queryset=Repository.objects.select_related('arch'), required=False, label=None, widget=FilteredSelectMultiple('Repos', False, attrs={'size': '30'})) diff --git a/operatingsystems/templates/operatingsystems/osvariant_detail.html b/operatingsystems/templates/operatingsystems/osvariant_detail.html index 0c1d306b..d4f011c6 100644 --- a/operatingsystems/templates/operatingsystems/osvariant_detail.html +++ b/operatingsystems/templates/operatingsystems/osvariant_detail.html @@ -24,7 +24,7 @@ Name {{ osvariant.name }} Architecture {{ osvariant.arch }} Codename {{ osvariant.codename }} - Hosts{{ osvariant.host_set.count }} + Hosts{{ osvariant.hosts_count }} OS Release{% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %} {% if user.is_authenticated and perms.is_admin %} diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 91b273aa..063a231c 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -40,7 +40,7 @@ def _get_filtered_osvariants(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - osvariants = OSVariant.objects.select_related() + osvariants = OSVariant.objects.select_related('osrelease', 'arch') if 'osrelease_id' in params: osvariants = osvariants.filter(osrelease=params['osrelease_id'][0]) @@ -60,7 +60,7 @@ def _get_filtered_osreleases(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - osreleases = OSRelease.objects.select_related() + osreleases = OSRelease.objects.all() if 'erratum_id' in params: osreleases = osreleases.filter(erratum=params['erratum_id'][0]) @@ -78,7 +78,7 @@ def _get_filtered_osreleases(filter_params): @login_required def osvariant_list(request): # Use cached hosts_count instead of expensive annotation - osvariants = OSVariant.objects.select_related().annotate( + osvariants = OSVariant.objects.select_related('osrelease', 'arch').annotate( repos_count=Count('osrelease__repos'), ) @@ -182,7 +182,7 @@ def delete_nohost_osvariants(request): @login_required def osrelease_list(request): - osreleases = OSRelease.objects.select_related() + osreleases = OSRelease.objects.all() if 'erratum_id' in request.GET: osreleases = osreleases.filter(erratum=request.GET['erratum_id']) @@ -347,7 +347,7 @@ class OSVariantViewSet(viewsets.ModelViewSet): """ API endpoint that allows operating system variants to be viewed or edited. """ - queryset = OSVariant.objects.all() + queryset = OSVariant.objects.select_related('osrelease', 'arch').all() serializer_class = OSVariantSerializer filterset_fields = ['name'] diff --git a/packages/managers.py b/packages/managers.py index c268f5cf..3a64e7ab 100644 --- a/packages/managers.py +++ b/packages/managers.py @@ -20,4 +20,4 @@ class PackageManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('name', 'arch') diff --git a/packages/utils.py b/packages/utils.py index 87395ff6..619407d2 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -281,7 +281,7 @@ def clean_packageupdates(): """ package_updates = list(PackageUpdate.objects.all()) for update in package_updates: - if update.host_set.count() == 0: + if not update.host_set.exists(): text = f'Removing unused PackageUpdate {update}' info_message(text=text) update.delete() @@ -325,7 +325,8 @@ def clean_packages(remove_duplicates=False): packagetype=package.packagetype, category=package.category, ) - if potential_duplicates.count() > 1: + potential_duplicates = list(potential_duplicates) + if len(potential_duplicates) > 1: for dupe in potential_duplicates: if dupe.id != package.id: info_message(text=f'Removing duplicate Package {dupe}') diff --git a/packages/views.py b/packages/views.py index 287c033d..9f75b415 100644 --- a/packages/views.py +++ b/packages/views.py @@ -32,7 +32,7 @@ @login_required def package_list(request): - packages = Package.objects.select_related() + packages = Package.objects.select_related('name', 'arch') if 'arch_id' in request.GET: packages = packages.filter(arch=request.GET['arch_id']).distinct() @@ -121,7 +121,7 @@ def package_list(request): @login_required def package_name_list(request): - packages = PackageName.objects.select_related() + packages = PackageName.objects.all() if 'arch_id' in request.GET: packages = packages.filter(package__arch=request.GET['arch_id']).distinct() @@ -165,7 +165,7 @@ def package_detail(request, package_id): @login_required def package_name_detail(request, packagename): package = get_object_or_404(PackageName, name=packagename) - allversions = Package.objects.select_related().filter(name=package.id) + allversions = Package.objects.select_related('name', 'arch').filter(name=package.id) return render(request, 'packages/package_name_detail.html', {'package': package, @@ -185,7 +185,7 @@ class PackageViewSet(viewsets.ModelViewSet): """ API endpoint that allows packages to be viewed or edited. """ - queryset = Package.objects.all() + queryset = Package.objects.select_related('name', 'arch').all() serializer_class = PackageSerializer filterset_fields = [ 'name', @@ -201,6 +201,6 @@ class PackageUpdateViewSet(viewsets.ModelViewSet): """ API endpoint that allows packages updates to be viewed or edited. """ - queryset = PackageUpdate.objects.all() + queryset = PackageUpdate.objects.select_related('oldpackage', 'newpackage').all() serializer_class = PackageUpdateSerializer filterset_fields = ['oldpackage', 'newpackage', 'security'] diff --git a/reports/views.py b/reports/views.py index 18f60aa6..1aa23913 100644 --- a/reports/views.py +++ b/reports/views.py @@ -45,7 +45,7 @@ def _get_filtered_reports(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - reports = Report.objects.select_related() + reports = Report.objects.all() if 'host_id' in params: reports = reports.filter(hostname=params['host_id'][0]) @@ -108,7 +108,7 @@ def upload(request): @login_required def report_list(request): - reports = Report.objects.select_related() + reports = Report.objects.all() if 'host_id' in request.GET: reports = reports.filter(hostname=request.GET['host_id']) diff --git a/repos/forms.py b/repos/forms.py index f9795b51..57ebae2a 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -28,7 +28,7 @@ class MirrorSelect2Widget(ModelSelect2MultipleWidget): model = Mirror search_fields = ['url__icontains', 'repo__name__icontains'] max_results = 50 - queryset = Mirror.objects.select_related().order_by('repo__name', 'url') + queryset = Mirror.objects.select_related('repo').order_by('repo__name', 'url') def __init__(self, *args, **kwargs): kwargs.setdefault('attrs', {}) @@ -41,7 +41,7 @@ def label_from_instance(self, obj): class EditRepoForm(ModelForm): mirrors = ModelMultipleChoiceField( - queryset=Mirror.objects.select_related().order_by('repo__name', 'url'), + queryset=Mirror.objects.select_related('repo').order_by('repo__name', 'url'), required=False, widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}), ) diff --git a/repos/managers.py b/repos/managers.py index 78f37a46..739c7ca3 100644 --- a/repos/managers.py +++ b/repos/managers.py @@ -20,4 +20,4 @@ class RepositoryManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('arch') diff --git a/repos/models.py b/repos/models.py index 421bca00..eec5f566 100644 --- a/repos/models.py +++ b/repos/models.py @@ -82,11 +82,11 @@ def refresh(self, force=False): force can be set to force a reset of all the mirrors metadata """ if force: - for mirror in self.mirror_set.all(): - mirror.packages_checksum = None - mirror.modules_checksum = None - mirror.errata_checksum = None - mirror.save() + self.mirror_set.all().update( + packages_checksum=None, + modules_checksum=None, + errata_checksum=None + ) if not self.auth_required: if self.repotype == Repository.DEB: @@ -108,9 +108,7 @@ def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors """ if force: - for mirror in self.mirror_set.all(): - mirror.errata_checksum = None - mirror.save() + self.mirror_set.all().update(errata_checksum=None) if self.repotype == Repository.RPM: refresh_repo_errata(self) @@ -120,10 +118,7 @@ def disable(self): each mirror so that it doesn't try to update its package metadata. """ self.enabled = False - for mirror in self.mirror_set.all(): - mirror.enabled = False - mirror.refresh = False - mirror.save() + self.mirror_set.all().update(enabled=False, refresh=False) def enable(self): """ Enable a repo. This involves enabling each mirror, which allows it @@ -131,10 +126,7 @@ def enable(self): mirror so that it updates its package metadata. """ self.enabled = True - for mirror in self.mirror_set.all(): - mirror.enabled = True - mirror.refresh = True - mirror.save() + self.mirror_set.all().update(enabled=True, refresh=True) class Mirror(models.Model): diff --git a/repos/utils.py b/repos/utils.py index 13cee149..0d81eb25 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -273,11 +273,12 @@ def find_best_repo(package, hostrepos): repo. Returns the best repo. """ best_repo = None - package_repos = hostrepos.filter(repo__mirror__packages=package).distinct() + package_repos = hostrepos.filter(repo__mirror__packages=package).select_related('repo').distinct() + package_repos = list(package_repos) if package_repos: best_repo = package_repos[0] - if package_repos.count() > 1: + if len(package_repos) > 1: for hostrepo in package_repos: if hostrepo.repo.security: best_repo = hostrepo diff --git a/repos/views.py b/repos/views.py index 58a9cdd9..4f6869ae 100644 --- a/repos/views.py +++ b/repos/views.py @@ -43,7 +43,7 @@ @login_required def repo_list(request): - repos = Repository.objects.select_related().order_by('name') + repos = Repository.objects.select_related('arch').order_by('name') if 'repotype' in request.GET: repos = repos.filter(repotype=request.GET['repotype']) @@ -146,11 +146,11 @@ def move_mirrors(repo): hostrepo.delete() mirror.repo = repo mirror.save() - if oldrepo.mirror_set.count() == 0: + if not oldrepo.mirror_set.exists(): oldrepo.delete() # Use cached packages_count instead of expensive annotation - mirrors = Mirror.objects.select_related().order_by('packages_checksum') + mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') checksum = None if 'checksum' in request.GET: @@ -318,9 +318,7 @@ def repo_edit(request, repo_id): repo = edit_form.save() repo.save() mirrors = edit_form.cleaned_data['mirrors'] - for mirror in mirrors: - mirror.repo = repo - mirror.save() + mirrors.update(repo=repo) if repo.enabled: repo.enable() else: @@ -418,7 +416,7 @@ def _get_filtered_repos(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - repos = Repository.objects.select_related().order_by('name') + repos = Repository.objects.select_related('arch').order_by('name') if 'repotype' in params: repos = repos.filter(repotype=params['repotype'][0]) @@ -509,7 +507,7 @@ def _get_filtered_mirrors(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - mirrors = Mirror.objects.select_related().order_by('packages_checksum') + mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') if 'checksum' in params: mirrors = mirrors.filter(packages_checksum=params['checksum'][0]) @@ -591,7 +589,7 @@ class RepositoryViewSet(viewsets.ModelViewSet): """ API endpoint that allows repositories to be viewed or edited. """ - queryset = Repository.objects.all() + queryset = Repository.objects.select_related('arch').all() serializer_class = RepositorySerializer @@ -599,7 +597,7 @@ class MirrorViewSet(viewsets.ModelViewSet): """ API endpoint that allows mirrors to be viewed or edited. """ - queryset = Mirror.objects.all() + queryset = Mirror.objects.select_related('repo').all() serializer_class = MirrorSerializer @@ -607,5 +605,5 @@ class MirrorPackageViewSet(viewsets.ModelViewSet): """ API endpoint that allows mirror packages to be viewed or edited. """ - queryset = MirrorPackage.objects.all() + queryset = MirrorPackage.objects.select_related('mirror', 'package').all() serializer_class = MirrorPackageSerializer diff --git a/security/managers.py b/security/managers.py index 4dfcffaf..f8d054a1 100644 --- a/security/managers.py +++ b/security/managers.py @@ -19,4 +19,4 @@ class CVEManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset() diff --git a/security/views.py b/security/views.py index ae56a82b..7ae5d851 100644 --- a/security/views.py +++ b/security/views.py @@ -32,7 +32,7 @@ @login_required def cwe_list(request): - cwes = CWE.objects.select_related() + cwes = CWE.objects.all() if 'search' in request.GET: terms = request.GET['search'].lower() @@ -65,7 +65,7 @@ def cwe_detail(request, cwe_id): @login_required def cve_list(request): - cves = CVE.objects.select_related() + cves = CVE.objects.all() if 'erratum_id' in request.GET: cves = cves.filter(erratum=request.GET['erratum_id']) @@ -117,7 +117,7 @@ def cve_detail(request, cve_id): @login_required def reference_list(request): - refs = Reference.objects.select_related().order_by('ref_type') + refs = Reference.objects.all().order_by('ref_type') if 'ref_type' in request.GET: refs = refs.filter(ref_type=request.GET['ref_type']).distinct() diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py index d22955e7..05985c48 100644 --- a/util/management/commands/revoke_api_key.py +++ b/util/management/commands/revoke_api_key.py @@ -42,7 +42,7 @@ def handle(self, *args, **options): if not api_keys.exists(): api_keys = APIKey.objects.filter(name=key_input) - if api_keys.count() == 0: + if not api_keys.exists(): raise CommandError(f'No API key found matching: {key_input}') elif api_keys.count() > 1: raise CommandError(f'Multiple keys match "{key_input}". Please be more specific.') diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 0fdeaff1..119e7f34 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -19,6 +19,7 @@ from urllib.parse import urlencode from django.template import Library +from django.db.models import Sum from django.template.loader import get_template from django.utils import timezone from django.utils.html import format_html @@ -138,7 +139,4 @@ def reports_timedelta(): @register.simple_tag def host_count(osrelease): - host_count = 0 - for osvariant in osrelease.osvariant_set.all(): - host_count += osvariant.host_set.count() - return host_count + return osrelease.osvariant_set.aggregate(total=Sum('hosts_count'))['total'] or 0 From a9bc4aeea0ffc1fbc53a12b17923cb9d751ed3d1 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:30:47 -0500 Subject: [PATCH 3/6] add .iterator() to large queryset loops --- hosts/tasks.py | 6 +++--- sbin/patchman | 6 +++--- security/tasks.py | 4 ++-- security/utils.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hosts/tasks.py b/hosts/tasks.py index a8632a5d..b9305cf9 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -33,7 +33,7 @@ def find_host_updates(host_id): def find_all_host_updates(): """ Task to find updates for all hosts """ - for host in Host.objects.all(): + for host in Host.objects.all().iterator(): find_host_updates.delay(host.id) @@ -43,7 +43,7 @@ def find_all_host_updates_homogenous(): """ updated_hosts = [] ts = get_datetime_now() - for host in Host.objects.all(): + for host in Host.objects.all().iterator(): if host not in updated_hosts: host.find_updates() host.updated_at = ts @@ -61,7 +61,7 @@ def find_all_host_updates_homogenous(): updates = host.updates.all() phosts = [] - for fhost in filtered_hosts: + for fhost in filtered_hosts.iterator(): frepos = set(fhost.repos.all()) if repos != frepos: continue diff --git a/sbin/patchman b/sbin/patchman index 5dabdfa7..0e164c98 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -164,7 +164,7 @@ def host_updates_alt(host=None): updated_hosts = [] hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() - for host in hosts: + for host in hosts.iterator(): info_message(text=str(host)) if host not in updated_hosts: host.find_updates() @@ -183,7 +183,7 @@ def host_updates_alt(host=None): updates = host.updates.all() phosts = [] - for fhost in filtered_hosts: + for fhost in filtered_hosts.iterator(): frepos = set(fhost.repos.all()) rdiff = repos.difference(frepos) @@ -352,7 +352,7 @@ def process_reports(host=None, force=False): info_message(text=text) - for report in reports: + for report in reports.iterator(): report.process(find_updates=False) diff --git a/security/tasks.py b/security/tasks.py index 0cfbc2f1..93d48427 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -50,7 +50,7 @@ def update_cves(): if cache.add(lock_key, 'true', lock_expire): try: - for cve in CVE.objects.all(): + for cve in CVE.objects.all().iterator(): update_cve.delay(cve.id) finally: cache.delete(lock_key) @@ -87,7 +87,7 @@ def update_cwes(): if cache.add(lock_key, 'true', lock_expire): try: - for cwe in CWE.objects.all(): + for cwe in CWE.objects.all().iterator(): update_cwe.delay(cwe.id) finally: cache.delete(lock_key) diff --git a/security/utils.py b/security/utils.py index 127f2c73..745cf1f8 100644 --- a/security/utils.py +++ b/security/utils.py @@ -42,7 +42,7 @@ def update_cves(cve_id=None, fetch_nist_data=False): cve = CVE.objects.get(cve_id=cve_id) cve.fetch_cve_data(fetch_nist_data, sleep_secs=0) else: - for cve in CVE.objects.all(): + for cve in CVE.objects.all().iterator(): cve.fetch_cve_data(fetch_nist_data) @@ -56,7 +56,7 @@ def update_cwes(cve_id=None): cwes = cve.cwes.all() else: cwes = CWE.objects.all() - for cwe in cwes: + for cwe in cwes.iterator(): cwe.fetch_cwe_data() From 36d5bea4c354b1e9f10943a89f3cf0d2042d3dd7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:40:39 -0500 Subject: [PATCH 4/6] address template double-count issues --- errata/templates/errata/erratum_detail.html | 10 +++++---- .../operatingsystems/osrelease_detail.html | 10 +++++---- .../osvariant_delete_multiple.html | 2 +- util/templates/dashboard.html | 8 +++++-- util/views.py | 22 +++++++++++++++++++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/errata/templates/errata/erratum_detail.html b/errata/templates/errata/erratum_detail.html index 2dbfceaf..3e622401 100644 --- a/errata/templates/errata/erratum_detail.html +++ b/errata/templates/errata/erratum_detail.html @@ -8,10 +8,11 @@ {% block content %} +{% with affected_count=erratum.affected_packages.count fixed_count=erratum.fixed_packages.count %}
@@ -22,8 +23,8 @@ Type {{ erratum.e_type }} Published Date{{ erratum.issue_date|date|default_if_none:'' }} Synopsis {{ erratum.synopsis }} - Packages Affected {{ erratum.affected_packages.count }} - Packages Fixed {{ erratum.fixed_packages.count }} + Packages Affected {{ affected_count }} + Packages Fixed {{ fixed_count }} OS Releases Affected @@ -78,5 +79,6 @@
+{% endwith %} {% endblock %} diff --git a/operatingsystems/templates/operatingsystems/osrelease_detail.html b/operatingsystems/templates/operatingsystems/osrelease_detail.html index 740b9c4b..be94f43d 100644 --- a/operatingsystems/templates/operatingsystems/osrelease_detail.html +++ b/operatingsystems/templates/operatingsystems/osrelease_detail.html @@ -12,6 +12,7 @@ {% block content %} +{% with osvariant_count=osrelease.osvariant_set.count repos_count=osrelease.repos.count %}
  • Details
  • Variants
    • @@ -26,8 +27,8 @@ Name{{ osrelease.name }} CPE Name{% if osrelease.cpe_name %}{{ osrelease.cpe_name }}{% endif %} Codename{% if osrelease.codename %}{{ osrelease.codename }}{% endif %} - OS Variants{{ osrelease.osvariant_set.count }} - Repositories{{ osrelease.repos.count }} + OS Variants{{ osvariant_count }} + Repositories{{ repos_count }} Hosts{{ host_count }} Errata{{ osrelease.erratum_set.count }} @@ -40,7 +41,7 @@
    - {% if osrelease.osvariant_set.count == 0 %} + {% if osvariant_count == 0 %} {{ osrelease }} has no Variants {% else %} {% gen_table osrelease.osvariant_set.select_related %} @@ -50,7 +51,7 @@
    - {% if osrelease.repos.count == 0 %} + {% if repos_count == 0 %} {{ osrelease }} has no Repositories {% else %} {% gen_table osrelease.repos.select_related %} @@ -68,5 +69,6 @@
    +{% endwith %} {% endblock %} diff --git a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html index f6d0373d..6887e0a7 100644 --- a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html +++ b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html @@ -24,7 +24,7 @@ {% for osvariant in osvariants %} {{ osvariant }} - {% if osvariant.host_set.count != None %} {{ osvariant.host_set.count }} {% else %} 0 {% endif %} + {{ osvariant.host_set.count|default:0 }} {% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %} {% endfor %} diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 5976adcb..7afc18fb 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -13,7 +13,7 @@ {% block content %} {% with count=noosrelease_osvariants.count %} - {% if noosrelease_osvariants.count > 0 %} + {% if count > 0 %}
    @@ -24,7 +24,7 @@ {% endwith %} {% with count=nohost_osvariants.count %} - {% if nohost_osvariants.count > 0 %} + {% if count > 0 %}
    @@ -244,4 +244,8 @@ {% endif %} {% endwith %} +{% if not has_issues %} +
    No issues found!
    +{% endif %} + {% endblock %} diff --git a/util/views.py b/util/views.py index 073188a1..6222546b 100644 --- a/util/views.py +++ b/util/views.py @@ -110,10 +110,32 @@ def dashboard(request): possible_mirrors[checksum] = checksums[checksum] continue + has_issues = ( + noosrelease_osvariants.exists() or + nohost_osvariants.exists() or + (norepo_osreleases is not None and norepo_osreleases.exists()) or + stale_hosts.exists() or + reboot_hosts.exists() or + secupdate_hosts.exists() or + bugupdate_hosts.exists() or + norepo_hosts.exists() or + diff_rdns_hosts.exists() or + failed_mirrors.exists() or + disabled_mirrors.exists() or + norefresh_mirrors.exists() or + failed_repos.exists() or + unused_repos.exists() or + nomirror_repos.exists() or + nohost_repos.exists() or + bool(possible_mirrors) or + norepo_packages.exists() + ) + return render( request, 'dashboard.html', {'site': site, + 'has_issues': has_issues, 'noosrelease_osvariants': noosrelease_osvariants, 'norepo_hosts': norepo_hosts, 'nohost_osvariants': nohost_osvariants, From ffb1771fe05bf349803265fd565939fdea10c9cb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:58:47 -0500 Subject: [PATCH 5/6] move imports to top-level --- hosts/views.py | 6 +++--- operatingsystems/views.py | 4 ++-- packages/utils.py | 1 + reports/views.py | 32 ++++++++++---------------------- repos/views.py | 7 +++---- util/context_processors.py | 4 ++-- util/templatetags/common.py | 3 +-- 7 files changed, 22 insertions(+), 35 deletions(-) diff --git a/hosts/views.py b/hosts/views.py index f905f8c6..f940c3a4 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -31,6 +33,7 @@ from hosts.models import Host, HostRepo from hosts.serializers import HostRepoSerializer, HostSerializer from hosts.tables import HostTable +from hosts.tasks import find_host_updates from operatingsystems.models import OSRelease, OSVariant from reports.models import Report from util import sanitize_filter_params @@ -39,7 +42,6 @@ def _get_filtered_hosts(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) hosts = Host.objects.select_related('osvariant', 'arch', 'domain') @@ -222,7 +224,6 @@ def host_delete(request, hostname): def host_find_updates(request, hostname): """ Find updates using a celery task """ - from hosts.tasks import find_host_updates host = get_object_or_404(Host, hostname=hostname) find_host_updates.delay(host.id) text = f'Finding updates for Host {host}' @@ -261,7 +262,6 @@ def host_bulk_action(request): name = Host._meta.verbose_name if count == 1 else Host._meta.verbose_name_plural if action == 'find_updates': - from hosts.tasks import find_host_updates for host in hosts: find_host_updates.delay(host.id) messages.success(request, f'Queued {count} {name} for update check') diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 063a231c..1a6b3fda 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Count, Q @@ -37,7 +39,6 @@ def _get_filtered_osvariants(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) osvariants = OSVariant.objects.select_related('osrelease', 'arch') @@ -57,7 +58,6 @@ def _get_filtered_osvariants(filter_params): def _get_filtered_osreleases(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) osreleases = OSRelease.objects.all() diff --git a/packages/utils.py b/packages/utils.py index 619407d2..23e522e5 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -19,6 +19,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.db import IntegrityError, transaction +from django.db.models import Count, Min from arch.models import PackageArchitecture from packages.models import ( diff --git a/reports/views.py b/reports/views.py index 1aa23913..e5d1d438 100644 --- a/reports/views.py +++ b/reports/views.py @@ -16,8 +16,9 @@ # along with Patchman. If not, see import json -from urllib.parse import unquote +from urllib.parse import parse_qs, unquote +from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -28,21 +29,26 @@ from django.views.decorators.csrf import csrf_exempt from django_tables2 import RequestConfig from rest_framework import status, viewsets +from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly from rest_framework.response import Response +from rest_framework_api_key.permissions import HasAPIKey from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from reports.models import Report -from reports.serializers import ReportUploadSerializer -from reports.tables import ReportTable +from reports.serializers import ReportSerializer, ReportUploadSerializer +from reports.tables import ( + ReportModuleTable, ReportPackageTable, ReportRepoTable, ReportTable, + ReportUpdateTable, +) +from reports.tasks import process_report from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar def _get_filtered_reports(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) reports = Report.objects.all() @@ -78,7 +84,6 @@ def upload(request): report = Report.objects.create() report.parse(data, meta) - from reports.tasks import process_report process_report.delay(report.id) if 'report' in data and data['report'] == 'true': @@ -159,10 +164,6 @@ def report_detail(request, report_id): # Add tables for Protocol 2 reports if report.protocol == '2': - from reports.tables import ( - ReportModuleTable, ReportPackageTable, ReportRepoTable, - ReportUpdateTable, - ) if report.has_packages: context['packages_table'] = ReportPackageTable(report.packages_parsed) if report.has_repos: @@ -183,7 +184,6 @@ def report_detail(request, report_id): def report_process(request, report_id): """ Process a report using a celery task """ - from reports.tasks import process_report report = get_object_or_404(Report, id=report_id) report.processed = False report.save() @@ -243,7 +243,6 @@ def report_bulk_action(request): name = Report._meta.verbose_name if count == 1 else Report._meta.verbose_name_plural if action == 'process': - from reports.tasks import process_report for report in reports: report.processed = False report.save() @@ -273,12 +272,6 @@ class ReportViewSet(viewsets.ViewSet): """ def get_permissions(self): - from django.conf import settings - from rest_framework.permissions import ( - AllowAny, IsAuthenticatedOrReadOnly, - ) - from rest_framework_api_key.permissions import HasAPIKey - # POST requires API key if configured, otherwise allow any if self.action == 'create': if getattr(settings, 'REQUIRE_API_KEY', False): @@ -289,16 +282,12 @@ def get_permissions(self): def list(self, request): """List all reports.""" - from reports.serializers import ReportSerializer queryset = Report.objects.all().order_by('-created') serializer = ReportSerializer(queryset, many=True, context={'request': request}) return Response(serializer.data) def retrieve(self, request, pk=None): """Retrieve a single report.""" - from django.shortcuts import get_object_or_404 - - from reports.serializers import ReportSerializer report = get_object_or_404(Report, pk=pk) serializer = ReportSerializer(report, context={'request': request}) return Response(serializer.data) @@ -357,7 +346,6 @@ def create(self, request): ) # Queue for async processing - from reports.tasks import process_report process_report.delay(report.id) return Response( diff --git a/repos/views.py b/repos/views.py index 4f6869ae..e49c4972 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db import IntegrityError @@ -36,6 +38,7 @@ MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, ) from repos.tables import MirrorTable, RepositoryTable +from repos.tasks import refresh_repo from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar @@ -403,7 +406,6 @@ def repo_toggle_security(request, repo_id): def repo_refresh(request, repo_id): """ Refresh a repo using a celery task """ - from repos.tasks import refresh_repo repo = get_object_or_404(Repository, id=repo_id) refresh_repo.delay(repo.id) text = f'Repostory {repo} is being refreshed' @@ -413,7 +415,6 @@ def repo_refresh(request, repo_id): def _get_filtered_repos(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) repos = Repository.objects.select_related('arch').order_by('name') @@ -486,7 +487,6 @@ def repo_bulk_action(request): repos.update(security=False) messages.success(request, f'Marked {count} {name} as non-security') elif action == 'refresh': - from repos.tasks import refresh_repo for repo in repos: refresh_repo.delay(repo.id) messages.success(request, f'Queued {count} {name} for refresh') @@ -504,7 +504,6 @@ def repo_bulk_action(request): def _get_filtered_mirrors(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') diff --git a/util/context_processors.py b/util/context_processors.py index a9c46545..97bbcf2c 100644 --- a/util/context_processors.py +++ b/util/context_processors.py @@ -16,6 +16,7 @@ import subprocess from datetime import timedelta +from importlib.metadata import version as get_pkg_version from pathlib import Path from django.db.models import F @@ -52,8 +53,7 @@ def _get_version(): """Get version from package metadata or VERSION.txt.""" # Try importlib.metadata first (for installed packages) try: - from importlib.metadata import version - return version('patchman') + return get_pkg_version('patchman') except Exception: pass diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 119e7f34..dc9f4fb5 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -18,8 +18,8 @@ from datetime import timedelta from urllib.parse import urlencode -from django.template import Library from django.db.models import Sum +from django.template import Library, engines from django.template.loader import get_template from django.utils import timezone from django.utils.html import format_html @@ -74,7 +74,6 @@ def gen_table(context, object_list, template_name=None): RequestConfig(request, paginate=False).configure(table) # Render using the table's configured template - from django.template import engines django_engine = engines['django'] template = django_engine.from_string('{% load django_tables2 %}{% render_table table %}') return template.render({'table': table, 'request': request}) From 0e0d834333652babe5aca9afe7ad3ed4ccdf3ce2 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 01:27:06 -0500 Subject: [PATCH 6/6] =?UTF-8?q?replace=20o(n=C2=B2)=20comparisons=20with?= =?UTF-8?q?=20id=20sets=20and=20sql=20aggregation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hosts/tasks.py | 31 ++++++++++----------- packages/utils.py | 70 +++++++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/hosts/tasks.py b/hosts/tasks.py index b9305cf9..71ec203a 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -41,10 +41,10 @@ def find_all_host_updates(): def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ - updated_hosts = [] + updated_host_ids = set() ts = get_datetime_now() for host in Host.objects.all().iterator(): - if host not in updated_hosts: + if host.id not in updated_host_ids: host.find_updates() host.updated_at = ts host.save() @@ -56,23 +56,20 @@ def find_all_host_updates_homogenous(): # and exclude hosts with the current timestamp filtered_hosts = filtered_hosts.exclude(updated_at=ts) - packages = set(host.packages.all()) - repos = set(host.repos.all()) - updates = host.updates.all() + package_ids = frozenset(host.packages.values_list('id', flat=True)) + repo_ids = frozenset(host.repos.values_list('id', flat=True)) + updates = list(host.updates.all()) - phosts = [] for fhost in filtered_hosts.iterator(): - frepos = set(fhost.repos.all()) - if repos != frepos: + frepo_ids = frozenset(fhost.repos.values_list('id', flat=True)) + if repo_ids != frepo_ids: continue - fpackages = set(fhost.packages.all()) - if packages != fpackages: + fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True)) + if package_ids != fpackage_ids: continue - phosts.append(fhost) - for phost in phosts: - phost.updates.set(updates) - phost.updated_at = ts - phost.save() - updated_hosts.append(phost) - info_message(text=f'Added the same updates to {phost}') + fhost.updates.set(updates) + fhost.updated_at = ts + fhost.save() + updated_host_ids.add(fhost.id) + info_message(text=f'Added the same updates to {fhost}') diff --git a/packages/utils.py b/packages/utils.py index 23e522e5..c7f86b4c 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -280,21 +280,30 @@ def get_matching_packages(name, epoch, version, release, p_type, arch=None): def clean_packageupdates(): """ Removes PackageUpdate objects that are no longer linked to any hosts """ - package_updates = list(PackageUpdate.objects.all()) - for update in package_updates: - if not update.host_set.exists(): - text = f'Removing unused PackageUpdate {update}' + orphaned = PackageUpdate.objects.filter(host__isnull=True) + for update in orphaned: + text = f'Removing unused PackageUpdate {update}' + info_message(text=text) + update.delete() + + duplicate_updates = PackageUpdate.objects.values( + 'oldpackage', 'newpackage', 'security' + ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1) + + for update in duplicate_updates: + extra_updates = PackageUpdate.objects.filter( + oldpackage=update['oldpackage'], + newpackage=update['newpackage'], + security=update['security'] + ).exclude(id=update['keep_id']) + keep_update = PackageUpdate.objects.get(id=update['keep_id']) + for extra_update in extra_updates: + text = f'Removing duplicate PackageUpdate: {extra_update}' info_message(text=text) - update.delete() - for duplicate in package_updates: - if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ - update.security == duplicate.security and update.id != duplicate.id: - text = f'Removing duplicate PackageUpdate: {update}' - info_message(text=text) - for host in duplicate.host_set.all(): - host.updates.remove(duplicate) - host.updates.add(update) - duplicate.delete() + for host in extra_update.host_set.all(): + host.updates.remove(extra_update) + host.updates.add(keep_update) + extra_update.delete() def clean_packages(remove_duplicates=False): @@ -316,22 +325,23 @@ def clean_packages(remove_duplicates=False): packages.delete() if remove_duplicates: info_message(text='Checking for duplicate Packages...') - for package in Package.objects.all(): - potential_duplicates = Package.objects.filter( - name=package.name, - arch=package.arch, - epoch=package.epoch, - version=package.version, - release=package.release, - packagetype=package.packagetype, - category=package.category, - ) - potential_duplicates = list(potential_duplicates) - if len(potential_duplicates) > 1: - for dupe in potential_duplicates: - if dupe.id != package.id: - info_message(text=f'Removing duplicate Package {dupe}') - dupe.delete() + duplicates = Package.objects.values( + 'name', 'arch', 'epoch', 'version', 'release', 'packagetype', 'category' + ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1) + + for dup in duplicates: + to_delete = Package.objects.filter( + name=dup['name'], + arch=dup['arch'], + epoch=dup['epoch'], + version=dup['version'], + release=dup['release'], + packagetype=dup['packagetype'], + category=dup['category'] + ).exclude(id=dup['keep_id']) + for package in to_delete: + info_message(text=f'Removing duplicate Package {package}') + package.delete() def clean_packagenames():