Skip to content

Commit 69c1ba3

Browse files
committed
Add project top contributors
1 parent b520639 commit 69c1ba3

21 files changed

+307
-31
lines changed

Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ dump-data:
1313
enrich-data: github-enrich-issues owasp-enrich-projects
1414

1515
exec-backend-command:
16-
@docker exec -i nest-backend $(CMD) 2>/dev/null
16+
@docker exec -i nest-backend $(CMD)
1717

1818
exec-backend-command-it:
1919
@docker exec -it nest-backend $(CMD) 2>/dev/null

backend/apps/github/admin.py

+10
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from apps.github.models.organization import Organization
99
from apps.github.models.release import Release
1010
from apps.github.models.repository import Repository
11+
from apps.github.models.repository_contributor import RepositoryContributor
1112
from apps.github.models.user import User
1213

1314

@@ -82,6 +83,14 @@ def custom_field_title(self, obj):
8283
custom_field_github_url.short_description = "GitHub 🔗"
8384

8485

86+
class RepositoryContributorAdmin(admin.ModelAdmin):
87+
autocomplete_fields = (
88+
"repository",
89+
"user",
90+
)
91+
search_fields = ("user__name",)
92+
93+
8594
class OrganizationAdmin(admin.ModelAdmin):
8695
list_display = (
8796
"title",
@@ -110,4 +119,5 @@ class UserAdmin(admin.ModelAdmin):
110119
admin.site.register(Organization, OrganizationAdmin)
111120
admin.site.register(Release, ReleaseAdmin)
112121
admin.site.register(Repository, RepositoryAdmin)
122+
admin.site.register(RepositoryContributor, RepositoryContributorAdmin)
113123
admin.site.register(User, UserAdmin)

backend/apps/github/common.py

+17-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from apps.github.models.organization import Organization
1010
from apps.github.models.release import Release
1111
from apps.github.models.repository import Repository
12+
from apps.github.models.repository_contributor import RepositoryContributor
1213
from apps.github.models.user import User
1314
from apps.github.utils import check_owasp_site_repository
1415

@@ -105,5 +106,20 @@ def sync_repository(gh_repository, organization=None, user=None):
105106
else None
106107
)
107108
releases.append(Release.update_data(gh_release, author=author, repository=repository))
109+
Release.bulk_save(releases)
110+
111+
# GitHub repository contributors.
112+
repository_contributors = []
113+
for gh_contributor in gh_repository.get_contributors():
114+
user = (
115+
User.update_data(gh_contributor)
116+
if gh_contributor and gh_contributor.type != "Bot"
117+
else None
118+
)
119+
if user:
120+
repository_contributors.append(
121+
RepositoryContributor.update_data(gh_contributor, repository=repository, user=user)
122+
)
123+
RepositoryContributor.bulk_save(repository_contributors)
108124

109-
return organization, repository, releases
125+
return organization, repository

backend/apps/github/constants.py

+3
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@
66
GITHUB_ITEMS_PER_PAGE = 100
77
GITHUB_REPOSITORY_RE = re.compile("^https://github.com/([^/]+)/([^/]+)(/.*)?$")
88
GITHUB_USER_RE = re.compile("^https://github.com/([^/]+)/?$")
9+
10+
OWASP_FOUNDATION_LOGIN = "OWASPFoundation"
11+
OWASP_LOGIN = "owasp"

backend/apps/github/management/commands/github_update_owasp_organization.py

+1-7
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
from apps.github.common import sync_repository
1111
from apps.github.constants import GITHUB_ITEMS_PER_PAGE
12-
from apps.github.models.release import Release
1312
from apps.github.models.repository import Repository
1413
from apps.owasp.constants import OWASP_ORGANIZATION_NAME
1514
from apps.owasp.models.chapter import Chapter
@@ -45,7 +44,6 @@ def handle(self, *_args, **options):
4544
committees = []
4645
events = []
4746
projects = []
48-
releases = []
4947

5048
offset = options["offset"]
5149
gh_repositories = gh_owasp_organization.get_repos(
@@ -59,10 +57,9 @@ def handle(self, *_args, **options):
5957
entity_key = gh_repository.name.lower()
6058
print(f"{prefix:<12} https://owasp.org/{entity_key}")
6159

62-
owasp_organization, repository, new_releases = sync_repository(
60+
owasp_organization, repository = sync_repository(
6361
gh_repository, organization=owasp_organization, user=owasp_user
6462
)
65-
releases.extend(new_releases)
6663

6764
# OWASP chapters.
6865
if entity_key.startswith("www-chapter-"):
@@ -80,9 +77,6 @@ def handle(self, *_args, **options):
8077
elif entity_key.startswith("www-committee-"):
8178
committees.append(Committee.update_data(gh_repository, repository, save=False))
8279

83-
# Bulk save data.
84-
Release.bulk_save(releases)
85-
8680
Chapter.bulk_save(chapters)
8781
Committee.bulk_save(committees)
8882
Event.bulk_save(events)

backend/apps/github/management/commands/github_update_related_repositories.py

+1-5
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from apps.github.common import sync_repository
1111
from apps.github.constants import GITHUB_ITEMS_PER_PAGE
1212
from apps.github.models.issue import Issue
13-
from apps.github.models.release import Release
1413
from apps.github.utils import get_repository_path
1514
from apps.owasp.models.project import Project
1615

@@ -30,7 +29,6 @@ def handle(self, *args, **options):
3029

3130
issues = []
3231
projects = []
33-
releases = []
3432

3533
offset = options["offset"]
3634
for idx, project in enumerate(active_projects[offset:]):
@@ -53,16 +51,14 @@ def handle(self, *args, **options):
5351
project.save(update_fields=("invalid_urls", "related_urls"))
5452
continue
5553

56-
organization, repository, new_releases = sync_repository(gh_repository)
54+
organization, repository = sync_repository(gh_repository)
5755
if organization is not None:
5856
organization.save()
5957

6058
project.repositories.add(repository)
61-
releases.extend(new_releases)
6259

6360
projects.append(project)
6461

6562
# Bulk save data.
6663
Issue.bulk_save(issues)
67-
Release.bulk_save(releases)
6864
Project.bulk_save(projects)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Generated by Django 5.1.1 on 2024-09-21 19:07
2+
3+
import django.db.models.deletion
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
dependencies = [
9+
("github", "0006_alter_issue_state_reason"),
10+
]
11+
12+
operations = [
13+
migrations.CreateModel(
14+
name="RepositoryContributor",
15+
fields=[
16+
(
17+
"id",
18+
models.BigAutoField(
19+
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
20+
),
21+
),
22+
("nest_created_at", models.DateTimeField(auto_now_add=True)),
23+
("nest_updated_at", models.DateTimeField(auto_now=True)),
24+
("node_id", models.CharField(unique=True, verbose_name="Node ID")),
25+
(
26+
"contributions_count",
27+
models.PositiveIntegerField(default=0, verbose_name="Contributions"),
28+
),
29+
(
30+
"repository",
31+
models.ForeignKey(
32+
on_delete=django.db.models.deletion.CASCADE,
33+
to="github.repository",
34+
verbose_name="Repository",
35+
),
36+
),
37+
(
38+
"user",
39+
models.ForeignKey(
40+
on_delete=django.db.models.deletion.CASCADE,
41+
to="github.user",
42+
verbose_name="User",
43+
),
44+
),
45+
],
46+
options={
47+
"verbose_name_plural": "Contributors",
48+
"db_table": "github_repository_contributors",
49+
},
50+
),
51+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Generated by Django 5.1.1 on 2024-09-21 19:09
2+
3+
from django.db import migrations
4+
5+
6+
class Migration(migrations.Migration):
7+
dependencies = [
8+
("github", "0007_repositorycontributor"),
9+
]
10+
11+
operations = [
12+
migrations.AlterModelOptions(
13+
name="repositorycontributor",
14+
options={"verbose_name_plural": "Repository contributors"},
15+
),
16+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Generated by Django 5.1.1 on 2024-09-21 19:13
2+
3+
from django.db import migrations
4+
5+
6+
class Migration(migrations.Migration):
7+
dependencies = [
8+
("github", "0008_alter_repositorycontributor_options"),
9+
]
10+
11+
operations = [
12+
migrations.RemoveField(
13+
model_name="repositorycontributor",
14+
name="node_id",
15+
),
16+
]

backend/apps/github/models/common.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,16 @@ class Meta:
3030
created_at = models.DateTimeField(verbose_name="Created at")
3131
updated_at = models.DateTimeField(verbose_name="Updated at")
3232

33+
@property
3334
def title(self):
34-
"""User title."""
35+
"""Entity title."""
3536
return f"{self.name or self.login}"
3637

38+
@property
39+
def url(self):
40+
"""Entity URL."""
41+
return f"https://github.com/{self.login.lower()}"
42+
3743
def from_github(self, data):
3844
"""Update instance based on GitHub data."""
3945
field_mapping = {

backend/apps/github/models/repository.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from github.GithubException import GithubException
88

99
from apps.common.models import TimestampedModel
10+
from apps.github.constants import OWASP_LOGIN
1011
from apps.github.models.common import NodeModel
1112
from apps.github.models.mixins import RepositoryIndexMixin
1213
from apps.github.utils import (
@@ -167,7 +168,7 @@ def from_github(
167168
# Key and OWASP repository flags.
168169
self.key = self.name.lower()
169170
self.is_owasp_repository = (
170-
organization is not None and organization.login.lower() == "owasp"
171+
organization is not None and organization.login.lower() == OWASP_LOGIN
171172
)
172173
self.is_owasp_site_repository = check_owasp_site_repository(self.key)
173174

@@ -207,7 +208,10 @@ def from_github(
207208
for target in targets if isinstance(targets, list) else [targets]:
208209
if not target:
209210
continue
210-
is_funding_policy_compliant = check_funding_policy_compliance(platform, target)
211+
is_funding_policy_compliant = check_funding_policy_compliance(
212+
platform,
213+
target,
214+
)
211215

212216
if not is_funding_policy_compliant:
213217
break
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"""Github app label model."""
2+
3+
from django.db import models
4+
from django.template.defaultfilters import pluralize
5+
6+
from apps.common.models import BulkSaveModel, TimestampedModel
7+
8+
TOP_CONTRIBUTORS_LIMIT = 20
9+
10+
11+
class RepositoryContributor(BulkSaveModel, TimestampedModel):
12+
"""Repository contributor model."""
13+
14+
class Meta:
15+
db_table = "github_repository_contributors"
16+
verbose_name_plural = "Repository contributors"
17+
18+
contributions_count = models.PositiveIntegerField(verbose_name="Contributions", default=0)
19+
20+
# FKs.
21+
repository = models.ForeignKey(
22+
"github.Repository",
23+
verbose_name="Repository",
24+
on_delete=models.CASCADE,
25+
)
26+
user = models.ForeignKey(
27+
"github.User",
28+
verbose_name="User",
29+
on_delete=models.CASCADE,
30+
)
31+
32+
def __str__(self):
33+
"""Repository contributor human readable representation."""
34+
return (
35+
f"{self.user} has made {self.contributions_count} "
36+
f"contribution{pluralize(self.contributions_count)} to {self.repository}"
37+
)
38+
39+
def from_github(self, gh_label):
40+
"""Update instance based on GitHub contributor data."""
41+
field_mapping = {
42+
"contributions_count": "contributions",
43+
}
44+
45+
# Direct fields.
46+
for model_field, gh_field in field_mapping.items():
47+
value = getattr(gh_label, gh_field)
48+
if value is not None:
49+
setattr(self, model_field, value)
50+
51+
@staticmethod
52+
def bulk_save(repository_contributors):
53+
"""Bulk save repository contributors."""
54+
BulkSaveModel.bulk_save(RepositoryContributor, repository_contributors)
55+
56+
@staticmethod
57+
def update_data(gh_contributor, repository, user, save=True):
58+
"""Update repository contributor data."""
59+
try:
60+
repository_contributor = RepositoryContributor.objects.get(
61+
repository=repository,
62+
user=user,
63+
)
64+
except RepositoryContributor.DoesNotExist:
65+
repository_contributor = RepositoryContributor(
66+
repository=repository,
67+
user=user,
68+
)
69+
repository_contributor.from_github(gh_contributor)
70+
71+
if save:
72+
repository_contributor.save()
73+
74+
return repository_contributor

backend/apps/owasp/admin.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,11 @@ class EventAdmin(admin.ModelAdmin):
3535

3636
class ProjectAdmin(admin.ModelAdmin):
3737
autocomplete_fields = (
38-
"owasp_repository",
3938
"organizations",
39+
"owasp_repository",
4040
"owners",
4141
"repositories",
42+
"top_contributors",
4243
)
4344
list_display = (
4445
"custom_field_name",
@@ -57,6 +58,7 @@ class ProjectAdmin(admin.ModelAdmin):
5758
"level",
5859
"type",
5960
)
61+
ordering = ("-created_at",)
6062
search_fields = (
6163
"description",
6264
"key",

backend/apps/owasp/api/search/project.py

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ def get_projects(query, attributes=None, limit=25):
1919
"idx_name",
2020
"idx_stars_count",
2121
"idx_summary",
22+
"idx_top_contributors",
2223
"idx_topics",
2324
"idx_type",
2425
"idx_updated_at",

0 commit comments

Comments
 (0)