Add search API

Author: arkid15r

.gitignore

@@ -10,3 +10,4 @@ __pycache__
 .vscode
 *.code-workspace
 /volumes
+staticfiles

Makefile

@@ -12,12 +12,18 @@ github-sync-owasp-organization:
 github-sync-related-repositories:
 	@CMD="poetry run python manage.py github_sync_related_repositories" $(MAKE) run-backend-command
 
+index:
+	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) run-backend-command
+
 migrate:
 	@CMD="poetry run python manage.py migrate" $(MAKE) run-backend-command
 
 migrations:
 	@CMD="poetry run python manage.py makemigrations" $(MAKE) run-backend-command
 
+migrations-merge:
+	@CMD="poetry run python manage.py makemigrations --merge" $(MAKE) run-backend-command
+
 owasp-scrape-site-data:
 	@CMD="poetry run python manage.py owasp_scrape_site_data" $(MAKE) run-backend-command
 
@@ -27,6 +33,9 @@ owasp-update-projects:
 pre-commit:
 	@pre-commit run -a
 
+purge-data:
+	@CMD="poetry run python manage.py purge_data" $(MAKE) run-backend-command
+
 run:
 	@docker compose up
 
@@ -44,3 +53,7 @@ sync:
 
 test:
 	@cd backend && poetry run pytest; cd ..
+
+update:
+	@$(MAKE) sync
+	@$(MAKE) index

backend/apps/common/management/__init__.py

@@ -0,0 +1 @@
+"""Common management module."""

backend/apps/common/management/commands/__init__.py

@@ -0,0 +1 @@
+"""Common management commands."""

backend/apps/common/management/commands/purge_data.py

@@ -0,0 +1,32 @@
+"""A command to purge OWASP Nest data."""
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+from apps.github.models import Issue, Label, Organization, Release, Repository, User
+from apps.owasp.models import Chapter, Committee, Event, Project
+
+BATCH_SIZE = 10
+
+
+class Command(BaseCommand):
+    help = "Purge OWASP Nest data."
+
+    def handle(self, *_args, **options):
+        with connection.cursor() as cursor:
+            models = (
+                Chapter,
+                Committee,
+                Event,
+                Issue,
+                Label,
+                Organization,
+                Project,
+                Release,
+                Repository,
+                User,
+            )
+
+            for model in models:
+                cursor.execute(f"TRUNCATE TABLE {model._meta.db_table} CASCADE")  # noqa: SLF001
+                print(f"Purged GitHub {model._meta.verbose_name_plural}")  # noqa: SLF001

backend/apps/common/models.py

@@ -14,9 +14,10 @@ def bulk_save(model, objects):
         """Bulk save objects."""
         model.objects.bulk_create(o for o in objects if not o.id)
         model.objects.bulk_update(
-            [o for o in objects if o.id],
+            (o for o in objects if o.id),
             fields=[field.name for field in model._meta.fields if not field.primary_key],  # noqa: SLF001
         )
+        objects.clear()
 
 
 class TimestampedModel(models.Model):

backend/apps/common/templates/base.html

@@ -0,0 +1,6 @@
+"""Common app utils."""
+
+
+def join_values(fields, delimiter=" "):
+    """Join non-empty field values using the delimiter."""
+    delimiter.join(field for field in fields if field)

backend/apps/common/utils.py

@@ -56,7 +56,9 @@ class RepositoryAdmin(admin.ModelAdmin):
         "is_owasp_site_repository",
         "has_funding_yml",
         "is_funding_policy_compliant",
+        "is_template",
         "is_fork",
+        "organization",
     )
     ordering = ("-created_at",)
     search_fields = ("name",)

backend/apps/github/admin.py

@@ -2,6 +2,7 @@
 
 import re
 
+GITHUB_GHOST_USER_LOGIN = "ghost"
 GITHUB_ITEMS_PER_PAGE = 100
-GITHUB_ORGANIZATION_RE = re.compile("^https://github.com/([^/]+)/?$")
-GITHUB_REPOSITORY_RE = re.compile("^https://github.com/([^/]+)/([^/]+)/?$")
+GITHUB_REPOSITORY_RE = re.compile("^https://github.com/([^/]+)/([^/]+)(/.*)?$")
+GITHUB_USER_RE = re.compile("^https://github.com/([^/]+)/?$")

backend/apps/github/constants.py

@@ -0,0 +1,93 @@
+"""GitHub app index."""
+
+from datetime import timedelta as td
+
+from algoliasearch_django import AlgoliaIndex
+from algoliasearch_django.decorators import register
+from django.db.models import Q
+from django.utils import timezone
+
+from apps.github.models.issue import Issue
+
+
+@register(Issue)
+class IssueIndex(AlgoliaIndex):
+    """Issue index."""
+
+    index_name = "issues"
+
+    fields = (
+        "idx_author_login",
+        "idx_author_name",
+        "idx_body",
+        "idx_comments_count",
+        "idx_created_at",
+        "idx_labels",
+        "idx_project_description",
+        "idx_project_level",
+        "idx_project_name",
+        "idx_project_tags",
+        "idx_repository_contributors_count",
+        "idx_repository_description",
+        "idx_repository_forks_count",
+        "idx_repository_languages",
+        "idx_repository_name",
+        "idx_repository_stars_count",
+        "idx_repository_topics",
+        "idx_title",
+        "idx_updated_at",
+        "idx_url",
+    )
+
+    settings = {
+        "minProximity": 3,
+        "attributeForDistinct": "idx_project_name",
+        "distinct": True,
+        "indexLanguages": ["en"],
+        "customRanking": [
+            "desc(idx_created_at)",
+            "desc(idx_updated_at)",
+            "desc(idx_comments_count)",
+            "desc(idx_repository_contributors_count)",
+            "desc(idx_repository_stars_count)",
+            "desc(idx_repository_forks_count)",
+        ],
+        "ranking": [
+            "typo",
+            "geo",
+            "words",
+            "filters",
+            "proximity",
+            "attribute",
+            "exact",
+            "custom",
+        ],
+        "searchableAttributes": [
+            "unordered(idx_labels, idx_repository_languages)",
+            "unordered(idx_title, idx_project_name, idx_repository_name)",
+            "unordered(idx_project_description, idx_repository_description)",
+            "unordered(idx_project_tags, idx_repository_topics)",
+            "unordered(idx_author_login, idx_author_name)",
+            "unordered(idx_body)",
+        ],
+    }
+
+    should_index = "is_indexable"
+
+    def get_queryset(self):
+        """Get queryset."""
+        # We index all unassigned issues and issues with no activity within 60 days.
+        return (
+            Issue.objects.select_related(
+                "repository",
+            )
+            .prefetch_related(
+                "assignees",
+                "labels",
+                "repository__project_set",
+            )
+            .filter(
+                Q(assignees__isnull=True)
+                | Q(assignees__isnull=False, updated_at__lte=timezone.now() - td(days=60))
+            )
+        )

backend/apps/github/index.py

@@ -6,12 +6,10 @@
 from django.core.management.base import BaseCommand
 
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
-from apps.github.models import Issue, Organization, Release, Repository, sync_repository
+from apps.github.models import Release, Repository, sync_repository
 from apps.owasp.constants import OWASP_ORGANIZATION_NAME
 from apps.owasp.models import Chapter, Committee, Event, Project
 
-BATCH_SIZE = 10
-
 
 class Command(BaseCommand):
     help = "Updates OWASP entities based on their GitHub data."
@@ -20,17 +18,6 @@ def add_arguments(self, parser):
         parser.add_argument("--offset", default=0, required=False, type=int)
 
     def handle(self, *_args, **options):
-        def save_data():
-            """Save data to DB."""
-            Organization.bulk_save(organizations)
-            Issue.bulk_save(issues)
-            Release.bulk_save(releases)
-
-            Chapter.bulk_save(chapters)
-            Committee.bulk_save(committees)
-            Event.bulk_save(events)
-            Project.bulk_save(projects)
-
         gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE)
         gh_owasp_organization = gh.get_organization(OWASP_ORGANIZATION_NAME)
         remote_owasp_repositories_count = gh_owasp_organization.public_repos
@@ -41,27 +28,23 @@ def save_data():
         chapters = []
         committees = []
         events = []
-        issues = []
-        organizations = []
         projects = []
         releases = []
 
         offset = options["offset"]
-        for idx, gh_repository in enumerate(
-            gh_owasp_organization.get_repos(
-                type="public",
-                sort="created",
-                direction="asc",
-            )[offset:]
-        ):
-            print(f"{idx + offset + 1:<4} {gh_repository.name}")
+        gh_repositories = gh_owasp_organization.get_repos(
+            type="public",
+            sort="created",
+            direction="desc",
+        )
+        total_count = gh_repositories.totalCount - offset
+        for idx, gh_repository in enumerate(gh_repositories[offset:]):
+            prefix = f"{idx + offset + 1} of {total_count}"
+            print(f"{prefix:<12} {gh_repository.name}")
 
             owasp_organization, repository, new_releases = sync_repository(
                 gh_repository, organization=owasp_organization, user=owasp_user
             )
-            if not owasp_organization.id:
-                owasp_organization.save()
-
             releases.extend(new_releases)
 
             entity_key = gh_repository.name.lower()
@@ -81,11 +64,13 @@ def save_data():
             elif entity_key.startswith("www-committee-"):
                 committees.append(Committee.update_data(gh_repository, repository, save=False))
 
-            if idx % BATCH_SIZE == 0:
-                save_data()
+        # Bulk save data.
+        Release.bulk_save(releases)
 
-        # Save remaining data.
-        save_data()
+        Chapter.bulk_save(chapters)
+        Committee.bulk_save(committees)
+        Event.bulk_save(events)
+        Project.bulk_save(projects)
 
         # Check repository counts.
         local_owasp_repositories_count = Repository.objects.filter(

backend/apps/github/management/commands/github_sync_owasp_organization.py

@@ -14,8 +14,6 @@
 
 logger = logging.getLogger(__name__)
 
-BATCH_SIZE = 10
-
 
 class Command(BaseCommand):
     help = "Updates OWASP entities based on their owasp.org data."
@@ -24,12 +22,6 @@ def add_arguments(self, parser):
         parser.add_argument("--offset", default=0, required=False, type=int)
 
     def handle(self, *args, **options):
-        def save_data():
-            """Save data to DB."""
-            Issue.bulk_save(issues)
-            Release.bulk_save(releases)
-            Project.bulk_save(projects)
-
         active_projects = Project.objects.filter(is_active=True).order_by("created_at")
         gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE)
 
@@ -39,9 +31,10 @@ def save_data():
 
         offset = options["offset"]
         for idx, project in enumerate(active_projects[offset:]):
-            print(f"{idx + offset + 1:<4}", project.owasp_url)
+            prefix = f"{idx + offset + 1} of {active_projects.count() - offset}"
+            print(f"{prefix:<12} {project.owasp_url}")
 
-            repository_urls = project.repositories_raw.copy()
+            repository_urls = project.related_urls.copy()
             for repository_url in repository_urls:
                 repository_path = get_repository_path(repository_url)
                 if not repository_path:
@@ -52,8 +45,9 @@ def save_data():
                     gh_repository = gh.get_repo(repository_path)
                 except UnknownObjectException as e:
                     if e.data["status"] == "404" and "Not Found" in e.data["message"]:
-                        project.repositories_raw.remove(repository_url)
-                        project.save(update_fields=("repositories_raw",))
+                        project.invalid_urls.add(repository_url)
+                        project.related_urls.remove(repository_url)
+                        project.save(update_fields=("invalid_urls", "related_urls"))
                         continue
 
                 organization, repository, new_releases = sync_repository(gh_repository)
@@ -65,8 +59,7 @@ def save_data():
 
             projects.append(project)
 
-            if idx % BATCH_SIZE == 0:
-                save_data()
-
-        # Save remaining data.
-        save_data()
+        # Bulk save data.
+        Issue.bulk_save(issues)
+        Release.bulk_save(releases)
+        Project.bulk_save(projects)

backend/apps/github/management/commands/github_sync_related_repositories.py

@@ -0,0 +1,21 @@
+# Generated by Django 5.1.1 on 2024-09-05 23:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("github", "0039_alter_issue_title"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="issue",
+            options={"ordering": ("-updated_at", "-state"), "verbose_name_plural": "Issues"},
+        ),
+        migrations.AlterField(
+            model_name="issue",
+            name="url",
+            field=models.URLField(default="", max_length=500, verbose_name="URL"),
+        ),
+    ]