Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved NestBot /events command #657

Merged
merged 36 commits into from
Mar 2, 2025
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
825baa1
Improved NestBot /events command
abhayymishraa Jan 29, 2025
797e7f8
pre-commit
abhayymishraa Jan 29, 2025
dd2b4a5
verified commit
abhayymishraaa Jan 31, 2025
0c601d1
Merge branch 'main' into feat/events
abhayymishraaa Jan 31, 2025
eee4b37
verified commit with chnages
abhayymishraa Feb 2, 2025
823f367
Added events to use graphql as a source of data
abhayymishraa Feb 4, 2025
3962eb1
removed the old event logic sync
abhayymishraa Feb 4, 2025
fc95c4d
fixed testcase syncing
abhayymishraa Feb 4, 2025
49e6dd0
Merge branch 'main' into feat/events
abhayymishraa Feb 4, 2025
8148738
Merge branch 'main' into feat/events
abhayymishraa Feb 4, 2025
0906959
Merge branch 'main' into feat/events
abhayymishraa Feb 4, 2025
79cb0e3
Merge branch 'main' into feat/events
abhayymishraa Feb 13, 2025
3a73341
Merge branch 'main' into feat/events
arkid15r Feb 14, 2025
71bef1d
Merge branch 'main' into pr/abhayymishraa/657
arkid15r Feb 14, 2025
4558d4d
Update code
arkid15r Feb 14, 2025
17a502d
Merge branch 'main' into feat/events
abhayymishraa Feb 14, 2025
27ea9c5
Merge branch 'main' into feat/events
abhayymishraa Feb 20, 2025
fafcf81
new migration and removed unnecessary things
abhayymishraa Feb 20, 2025
a3f7c90
fixed some cases one remaining
abhayymishraa Feb 20, 2025
7482cb4
Merge branch 'main' into feat/events
abhayymishraa Feb 20, 2025
e951eb0
removed extra fields from the serializer
abhayymishraa Feb 20, 2025
98f6659
Merge branch 'main' into feat/events
arkid15r Feb 22, 2025
91271d1
Merge branch 'main' into feat/events
abhayymishraa Feb 22, 2025
52c180a
Merge branch 'main' into feat/events
abhayymishraa Feb 22, 2025
b5d0390
Merge branch 'main' into feat/events
abhayymishraa Feb 28, 2025
9b37d90
Update backend/apps/owasp/models/event.py
abhayymishraa Feb 28, 2025
216a6da
resolved suggestions
abhayymishraa Feb 28, 2025
337173c
fix testcase
abhayymishraa Feb 28, 2025
23f4cc1
Merge branch 'main' into feat/events
abhayymishraa Mar 1, 2025
ef3929f
Merge branch 'main' into feat/events
abhayymishraa Mar 1, 2025
68d4d84
resolved suggestion
abhayymishraa Mar 1, 2025
29cd4a2
Merge branch 'main' into feat/events
abhayymishraa Mar 2, 2025
ba75fb8
Merge branch 'main' into feat/events
abhayymishraa Mar 2, 2025
e6d7be4
Update code
arkid15r Mar 2, 2025
e6c3e1c
Apply suggestion
arkid15r Mar 2, 2025
5aa4f51
Merge branch 'main' into feat/events
arkid15r Mar 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ owasp-scrape-projects:
@echo "Scraping OWASP site projects data"
@CMD="poetry run python manage.py owasp_scrape_projects" $(MAKE) exec-backend-command

owasp-update-events:
@echo "Getting OWASP events data"
@CMD="poetry run python manage.py owasp_update_events" $(MAKE) exec-backend-command

poetry-update:
@CMD="poetry update" $(MAKE) exec-backend-command

Expand Down Expand Up @@ -116,4 +120,5 @@ update-data: \
owasp-scrape-committees \
owasp-scrape-projects \
github-update-project-related-repositories \
owasp-aggregate-projects
owasp-aggregate-projects \
owasp-update-events
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from apps.owasp.constants import OWASP_ORGANIZATION_NAME
from apps.owasp.models.chapter import Chapter
from apps.owasp.models.committee import Committee
from apps.owasp.models.event import Event
from apps.owasp.models.project import Project

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -48,7 +47,6 @@ def handle(self, *_args, **options):

chapters = []
committees = []
events = []
projects = []

offset = options["offset"]
Expand Down Expand Up @@ -82,17 +80,12 @@ def handle(self, *_args, **options):
elif entity_key.startswith("www-project-"):
projects.append(Project.update_data(gh_repository, repository, save=False))

# OWASP events.
elif entity_key.startswith("www-event-"):
events.append(Event.update_data(gh_repository, repository, save=False))

# OWASP committees.
elif entity_key.startswith("www-committee-"):
committees.append(Committee.update_data(gh_repository, repository, save=False))

Chapter.bulk_save(chapters)
Committee.bulk_save(committees)
Event.bulk_save(events)
Project.bulk_save(projects)

# Check repository counts.
Expand Down
1 change: 0 additions & 1 deletion backend/apps/owasp/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ class CommitteeAdmin(admin.ModelAdmin):


class EventAdmin(admin.ModelAdmin):
autocomplete_fields = ("owasp_repository",)
list_display = ("name",)
search_fields = ("name",)

Expand Down
2 changes: 0 additions & 2 deletions backend/apps/owasp/api/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ class Meta:
"name",
"description",
"url",
"created_at",
"updated_at",
)


Expand Down
21 changes: 21 additions & 0 deletions backend/apps/owasp/graphql/nodes/event.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
"""OWASP event GraphQL node."""

from apps.common.graphql.nodes import BaseNode
from apps.owasp.models.event import Event


class EventNode(BaseNode):
"""Event node."""

class Meta:
model = Event
fields = (
"category",
"category_description",
"end_date",
"description",
"key",
"name",
"start_date",
"url",
)
3 changes: 2 additions & 1 deletion backend/apps/owasp/graphql/queries/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

from .chapter import ChapterQuery
from .committee import CommitteeQuery
from .event import EventQuery
from .project import ProjectQuery
from .stats import StatsQuery


class OwaspQuery(ChapterQuery, CommitteeQuery, ProjectQuery, StatsQuery):
class OwaspQuery(ChapterQuery, CommitteeQuery, EventQuery, ProjectQuery, StatsQuery):
"""OWASP queries."""
23 changes: 23 additions & 0 deletions backend/apps/owasp/graphql/queries/event.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""OWASP event GraphQL queries."""

from datetime import datetime, timezone

import graphene

from apps.common.graphql.queries import BaseQuery
from apps.owasp.graphql.nodes.event import EventNode
from apps.owasp.models.event import Event


class EventQuery(BaseQuery):
"""Event queries."""

events = graphene.List(EventNode)

def resolve_events(root, info):
"""Resolve all events."""
today = datetime.now(timezone.utc).date()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use timezone.now() from django.utils


base_query = Event.objects.exclude(start_date__isnull=True).filter(start_date__gte=today)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

start_date should not be nullable


return base_query.order_by("start_date")
69 changes: 69 additions & 0 deletions backend/apps/owasp/management/commands/owasp_update_events.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""A command to update OWASP events."""

import yaml
from django.core.management.base import BaseCommand
from django.utils.text import slugify

from apps.github.utils import get_repository_file_content, normalize_url
from apps.owasp.models.event import Event, EventCategory
from apps.owasp.utils import parse_event_dates


class Command(BaseCommand):
help = "Import events from the provided YAML file"

def handle(self, *args, **kwargs):
url = "https://raw.githubusercontent.com/OWASP/owasp.github.io/main/_data/events.yml"
yaml_content = get_repository_file_content(url)
data = yaml.safe_load(yaml_content)
events = []

for category in data:
category_name = category.get("category", "")
category_description = category.get("description", "")

for event_data in category["events"]:
event_name_slug = slugify(event_data.get("name", ""))
key = event_name_slug
end_date = parse_event_dates(
event_data.get("dates", ""), event_data.get("start-date")
)

fields = {
"category": get_event_category(category_name),
"category_description": category_description,
"end_date": end_date,
"key": key,
"name": event_data.get("name", ""),
"description": event_data.get("optional-text", ""),
"start_date": event_data.get("start-date", None),
"url": normalize_url(event_data.get("url", "")) or "",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

❓ Verification inconclusive

Ensure start_date is properly parsed.

The start_date is directly set from the YAML data without parsing, but the Event model likely expects a datetime object.

                    "key": key,
                    "name": event_data.get("name", ""),
                    "description": event_data.get("optional-text", ""),
-                    "start_date": event_data.get("start-date", None),
+                    "start_date": parser.parse(event_data.get("start-date")).date() if event_data.get("start-date") else None,
                    "url": normalize_url(event_data.get("url", "")) or "",

🏁 Script executed:

#!/bin/bash
# Check the Event model definition to confirm start_date field type
rg "start_date" -A 1 -B 1 "apps/owasp/models/event.py"

Length of output: 119


Attention: Confirm that the Event model accepts a date object

The change correctly converts the YAML string into a date object using parser.parse(...).date(). However, since we could not locate the Event model (the file apps/owasp/models/event.py wasn’t found), please manually verify that the start_date field in the Event model indeed expects a date object. If the field requires a full datetime, you may need to remove the .date() conversion.

  • File & Lines: backend/apps/owasp/management/commands/owasp_update_events.py (original lines 37-38)
  • Diff snippet:
                        "start_date": event_data.get("start-date", None),
    -                    "start_date": event_data.get("start-date", None),
    +                    "start_date": parser.parse(event_data.get("start-date")).date() if event_data.get("start-date") else None,
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
"start_date": event_data.get("start-date", None),
"url": normalize_url(event_data.get("url", "")) or "",
"key": key,
"name": event_data.get("name", ""),
"description": event_data.get("optional-text", ""),
"start_date": parser.parse(event_data.get("start-date")).date() if event_data.get("start-date") else None,
"url": normalize_url(event_data.get("url", "")) or "",

}

try:
event = Event.objects.get(key=key)
# Update existing event
for key, value in fields.items():
setattr(event, key, value)
events.append(event)
except Event.DoesNotExist:
# Create new event
event = Event(**fields)
events.append(event)

if events:
self.stdout.write(f"Saving {len(events)} events...")
Event.bulk_save(events, fields)
self.stdout.write(self.style.SUCCESS("Successfully saved events"))
else:
self.stdout.write(self.style.WARNING("No events to save"))


def get_event_category(category_name):
"""Get event category."""
category_map = {
"Global": EventCategory.GLOBAL,
"AppSec Days": EventCategory.APPSEC_DAYS,
"Partner": EventCategory.PARTNER,
}
return category_map.get(category_name, EventCategory.OTHER)
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Generated by Django 5.1.5 on 2025-02-20 16:06

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("owasp", "0014_project_custom_tags"),
]

operations = [
migrations.RemoveField(
model_name="event",
name="created_at",
),
migrations.RemoveField(
model_name="event",
name="has_active_repositories",
),
migrations.RemoveField(
model_name="event",
name="is_active",
),
migrations.RemoveField(
model_name="event",
name="level",
),
migrations.RemoveField(
model_name="event",
name="owasp_repository",
),
migrations.RemoveField(
model_name="event",
name="summary",
),
migrations.RemoveField(
model_name="event",
name="tags",
),
migrations.RemoveField(
model_name="event",
name="topics",
),
migrations.RemoveField(
model_name="event",
name="updated_at",
),
migrations.AddField(
model_name="event",
name="category",
field=models.CharField(
choices=[
("global", "Global"),
("appsec_days", "AppSec Days"),
("partner", "Partner"),
("other", "Other"),
],
default="other",
max_length=20,
verbose_name="Category",
),
),
migrations.AddField(
model_name="event",
name="category_description",
field=models.TextField(blank=True, default="", verbose_name="Category Description"),
),
migrations.AddField(
model_name="event",
name="end_date",
field=models.DateField(blank=True, null=True, verbose_name="End Date"),
),
migrations.AddField(
model_name="event",
name="start_date",
field=models.DateField(blank=True, default="2025-01-01", verbose_name="Start Date"),
),
migrations.AlterField(
model_name="event",
name="description",
field=models.TextField(blank=True, default="", verbose_name="Description"),
),
]
58 changes: 24 additions & 34 deletions backend/apps/owasp/models/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,46 @@
from django.db import models

from apps.common.models import BulkSaveModel, TimestampedModel
from apps.owasp.models.common import RepositoryBasedEntityModel


class Event(BulkSaveModel, RepositoryBasedEntityModel, TimestampedModel):
class EventCategory(models.TextChoices):
"""Event category choices."""

GLOBAL = "global", "Global"
APPSEC_DAYS = "appsec_days", "AppSec Days"
PARTNER = "partner", "Partner"
OTHER = "other", "Other"


class Event(BulkSaveModel, TimestampedModel):
"""Event model."""

class Meta:
db_table = "owasp_events"
verbose_name_plural = "Events"

level = models.CharField(verbose_name="Level", max_length=5, default="", blank=True)
url = models.URLField(verbose_name="URL", default="", blank=True)
category = models.CharField(
verbose_name="Category",
max_length=20,
choices=EventCategory.choices,
default=EventCategory.OTHER,
)

owasp_repository = models.ForeignKey(
"github.Repository", on_delete=models.SET_NULL, blank=True, null=True
category_description = models.TextField(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can remove this field. The description should belong to category entity, not to the event as it's always tied to the category.

verbose_name="Category Description", default="", blank=True
)
end_date = models.DateField(verbose_name="End Date", null=True, blank=True)
key = models.CharField(verbose_name="Key", max_length=100, unique=True)
name = models.CharField(verbose_name="Name", max_length=100)
description = models.TextField(verbose_name="Description", default="", blank=True)
start_date = models.DateField(verbose_name="Start Date", default="2025-01-01", blank=True)
url = models.URLField(verbose_name="URL", default="", blank=True)

def __str__(self):
"""Event human readable representation."""
return f"{self.name or self.key}"

def from_github(self, repository):
"""Update instance based on GitHub repository data."""
field_mapping = {
"description": "pitch",
"level": "level",
"name": "title",
"tags": "tags",
}
RepositoryBasedEntityModel.from_github(self, field_mapping, repository)

# FKs.
self.owasp_repository = repository

@staticmethod
def bulk_save(events, fields=None):
"""Bulk save events."""
BulkSaveModel.bulk_save(Event, events, fields=fields)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe you should keep this method but just update its logic based on the new requirements.

@staticmethod
def update_data(gh_repository, repository, save=True):
"""Update event data."""
key = gh_repository.name.lower()
try:
event = Event.objects.get(key=key)
except Event.DoesNotExist:
event = Event(key=key)

event.from_github(repository)
if save:
event.save()

return event
Loading