Skip to content

Commit 3b8a61c

Browse files
jk-ozlabsjk-ozlabs
committed
Add patch tag infrastructure
This change add patch 'tags', eg 'Acked-by' / 'Reviewed-by', etc., to patchwork. Tag parsing is implemented in the patch parser's extract_tags function, which returns a Counter object of the tags in a comment. These are stored in the PatchTag (keyed to Tag) objects associated with each patch. We need to ensure that the main patch lists do not cause per-patch queries on the Patch.tags ManyToManyField (this would result in ~500 queries per page), so we introduce a new QuerySet (and Manager) for Patch, adding a with_tag_counts() method to populate the tag counts in a single query. As users may be migrating from previous patchwork versions (ie, with no tag counts in the database), we add a 'retag' management command. Signed-off-by: Jeremy Kerr <jk@ozlabs.org>
1 parent daa3ae4 commit 3b8a61c

17 files changed

Lines changed: 469 additions & 8 deletions

File tree

‎docs/INSTALL‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,12 @@ in brackets):
148148

149149
PYTHONPATH=lib/python ./manage.py collectstatic
150150

151-
and add privileges for your mail and web users. This is only needed if
151+
If you'd like to use the default tag set (Acked-by, Reviewed-by and
152+
Tested-by), then load these default tags:
153+
154+
PYTHONPATH=lib/python ./manage.py loaddata default_tags
155+
156+
Finally, add privileges for your mail and web users. This is only needed if
152157
you use the ident-based approach. If you use password-based database
153158
authentication, you can skip this step.
154159

‎lib/sql/grant-all.mysql.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,17 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_bundlepatch TO 'www-data'@loca
2323
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patch TO 'www-data'@localhost;
2424
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_emailoptout TO 'www-data'@localhost;
2525
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchchangenotification TO 'www-data'@localhost;
26+
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_tag TO 'www-data'@localhost;
27+
GRANT SELECT, UPDATE, INSERT, DELETE ON patchwork_patchtag TO 'www-data'@localhost;
2628

2729
-- allow the mail user (in this case, 'nobody') to add patches
2830
GRANT INSERT, SELECT ON patchwork_patch TO 'nobody'@localhost;
2931
GRANT INSERT, SELECT ON patchwork_comment TO 'nobody'@localhost;
3032
GRANT INSERT, SELECT ON patchwork_person TO 'nobody'@localhost;
33+
GRANT INSERT, SELECT, UPDATE, DELETE ON patchwork_patchtag TO 'nobody'@localhost;
3134
GRANT SELECT ON patchwork_project TO 'nobody'@localhost;
3235
GRANT SELECT ON patchwork_state TO 'nobody'@localhost;
36+
GRANT SELECT ON patchwork_tag TO 'nobody'@localhost;
3337

3438
COMMIT;
3539

‎lib/sql/grant-all.postgres.sql‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ GRANT SELECT, UPDATE, INSERT, DELETE ON
2323
patchwork_bundlepatch,
2424
patchwork_patch,
2525
patchwork_emailoptout,
26-
patchwork_patchchangenotification
26+
patchwork_patchchangenotification,
27+
patchwork_tag,
28+
patchwork_patchtag
2729
TO "www-data";
2830
GRANT SELECT, UPDATE ON
2931
auth_group_id_seq,
@@ -44,7 +46,9 @@ GRANT SELECT, UPDATE ON
4446
patchwork_state_id_seq,
4547
patchwork_emailconfirmation_id_seq,
4648
patchwork_userprofile_id_seq,
47-
patchwork_userprofile_maintainer_projects_id_seq
49+
patchwork_userprofile_maintainer_projects_id_seq,
50+
patchwork_tag_id_seq,
51+
patchwork_patchtag_id_seq
4852
TO "www-data";
4953

5054
-- allow the mail user (in this case, 'nobody') to add patches
@@ -53,14 +57,19 @@ GRANT INSERT, SELECT ON
5357
patchwork_comment,
5458
patchwork_person
5559
TO "nobody";
60+
GRANT INSERT, SELECT, UPDATE, DELETE ON
61+
patchwork_patchtag
62+
TO "nobody";
5663
GRANT SELECT ON
5764
patchwork_project,
58-
patchwork_state
65+
patchwork_state,
66+
patchwork_tag
5967
TO "nobody";
6068
GRANT UPDATE, SELECT ON
6169
patchwork_patch_id_seq,
6270
patchwork_person_id_seq,
63-
patchwork_comment_id_seq
71+
patchwork_comment_id_seq,
72+
patchwork_patchtag_id_seq
6473
TO "nobody";
6574

6675
COMMIT;

‎lib/sql/migration/015-add-patch-tags.sql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
BEGIN;
2+
ALTER TABLE patchwork_project ADD COLUMN use_tags boolean default true;
3+
4+
CREATE TABLE "patchwork_tag" (
5+
"id" serial NOT NULL PRIMARY KEY,
6+
"name" varchar(20) NOT NULL,
7+
"pattern" varchar(50) NOT NULL,
8+
"abbrev" varchar(2) NOT NULL UNIQUE
9+
);
10+
11+
CREATE TABLE "patchwork_patchtag" (
12+
"id" serial NOT NULL PRIMARY KEY,
13+
"patch_id" integer NOT NULL,
14+
"tag_id" integer NOT NULL REFERENCES "patchwork_tag" ("id"),
15+
"count" integer NOT NULL,
16+
UNIQUE ("patch_id", "tag_id")
17+
);
18+
19+
COMMIT;

‎patchwork/admin.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from django.contrib import admin
22
from patchwork.models import Project, Person, UserProfile, State, Patch, \
3-
Comment, Bundle
3+
Comment, Bundle, Tag
44

55
class ProjectAdmin(admin.ModelAdmin):
66
list_display = ('name', 'linkname','listid', 'listemail')
@@ -48,3 +48,8 @@ class BundleAdmin(admin.ModelAdmin):
4848
list_filter = ('public', 'project')
4949
search_fields = ('name', 'owner')
5050
admin.site.register(Bundle, BundleAdmin)
51+
52+
class TagAdmin(admin.ModelAdmin):
53+
list_display = ('name',)
54+
admin.site.register(Tag, TagAdmin)
55+

‎patchwork/fixtures/default_tags.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<django-objects version="1.0">
3+
<object pk="1" model="patchwork.tag">
4+
<field type="CharField" name="name">Acked-by</field>
5+
<field type="CharField" name="pattern">^Acked-by:</field>
6+
<field type="CharField" name="abbrev">A</field>
7+
</object>
8+
<object pk="2" model="patchwork.tag">
9+
<field type="CharField" name="name">Reviewed-by</field>
10+
<field type="CharField" name="pattern">^Reviewed-by:</field>
11+
<field type="CharField" name="abbrev">R</field>
12+
</object>
13+
<object pk="3" model="patchwork.tag">
14+
<field type="CharField" name="name">Tested-by</field>
15+
<field type="CharField" name="pattern">^Tested-by:</field>
16+
<field type="CharField" name="abbrev">T</field>
17+
</object>
18+
</django-objects>

‎patchwork/management/__init__.py‎

Whitespace-only changes.

‎patchwork/management/commands/__init__.py‎

Whitespace-only changes.

‎patchwork/management/commands/retag.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
2+
from django.core.management.base import BaseCommand, CommandError
3+
from patchwork.models import Patch
4+
import sys
5+
6+
class Command(BaseCommand):
7+
help = 'Update the tag (Ack/Review/Test) counts on existing patches'
8+
args = '[<patch_id>...]'
9+
10+
def handle(self, *args, **options):
11+
12+
qs = Patch.objects
13+
14+
if args:
15+
qs = qs.filter(id__in = args)
16+
17+
count = qs.count()
18+
i = 0
19+
20+
for patch in qs.iterator():
21+
patch.refresh_tag_counts()
22+
i += 1
23+
if (i % 10) == 0 or i == count:
24+
sys.stdout.write('%06d/%06d\r' % (i, count))
25+
sys.stdout.flush()
26+
sys.stderr.write('\ndone\n')

‎patchwork/models.py‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@
2222
from django.core.urlresolvers import reverse
2323
from django.contrib.sites.models import Site
2424
from django.conf import settings
25-
from patchwork.parser import hash_patch
25+
from django.utils.functional import cached_property
26+
from patchwork.parser import hash_patch, extract_tags
2627

2728
import re
2829
import datetime, time
2930
import random
31+
from collections import Counter, OrderedDict
3032

3133
class Person(models.Model):
3234
email = models.CharField(max_length=255, unique = True)
@@ -56,6 +58,7 @@ class Project(models.Model):
5658
scm_url = models.CharField(max_length=2000, blank=True)
5759
webscm_url = models.CharField(max_length=2000, blank=True)
5860
send_notifications = models.BooleanField(default=False)
61+
use_tags = models.BooleanField(default=True)
5962

6063
def __unicode__(self):
6164
return self.name
@@ -65,6 +68,12 @@ def is_editable(self, user):
6568
return False
6669
return self in user.profile.maintainer_projects.all()
6770

71+
@cached_property
72+
def tags(self):
73+
if not self.use_tags:
74+
return []
75+
return list(Tag.objects.all())
76+
6877
class Meta:
6978
ordering = ['linkname']
7079

@@ -165,9 +174,68 @@ def _construct(string = ''):
165174
def db_type(self, connection=None):
166175
return 'char(%d)' % self.n_bytes
167176

177+
class Tag(models.Model):
178+
name = models.CharField(max_length=20)
179+
pattern = models.CharField(max_length=50,
180+
help_text='A simple regex to match the tag in the content of '
181+
'a message. Will be used with MULTILINE and IGNORECASE '
182+
'flags. eg. ^Acked-by:')
183+
abbrev = models.CharField(max_length=2, unique=True,
184+
help_text='Short (one-or-two letter) abbreviation for the tag, '
185+
'used in table column headers')
186+
187+
def __unicode__(self):
188+
return self.name
189+
190+
@property
191+
def attr_name(self):
192+
return 'tag_%d_count' % self.id
193+
194+
class Meta:
195+
ordering = ['abbrev']
196+
197+
class PatchTag(models.Model):
198+
patch = models.ForeignKey('Patch')
199+
tag = models.ForeignKey('Tag')
200+
count = models.IntegerField(default=1)
201+
202+
class Meta:
203+
unique_together = [('patch', 'tag')]
204+
168205
def get_default_initial_patch_state():
169206
return State.objects.get(ordering=0)
170207

208+
class PatchQuerySet(models.query.QuerySet):
209+
210+
def with_tag_counts(self, project):
211+
if not project.use_tags:
212+
return self
213+
214+
# We need the project's use_tags field loaded for Project.tags().
215+
# Using prefetch_related means we'll share the one instance of
216+
# Project, and share the project.tags cache between all patch.project
217+
# references.
218+
qs = self.prefetch_related('project')
219+
select = OrderedDict()
220+
select_params = []
221+
for tag in project.tags:
222+
select[tag.attr_name] = ("coalesce("
223+
"(SELECT count FROM patchwork_patchtag "
224+
"WHERE patchwork_patchtag.patch_id=patchwork_patch.id "
225+
"AND patchwork_patchtag.tag_id=%s), 0)")
226+
select_params.append(tag.id)
227+
228+
return qs.extra(select=select, select_params=select_params)
229+
230+
class PatchManager(models.Manager):
231+
use_for_related_fields = True
232+
233+
def get_queryset(self):
234+
return PatchQuerySet(self.model, using=self.db)
235+
236+
def with_tag_counts(self, project):
237+
return self.get_queryset().with_tag_counts(project)
238+
171239
class Patch(models.Model):
172240
project = models.ForeignKey(Project)
173241
msgid = models.CharField(max_length=255)
@@ -182,13 +250,34 @@ class Patch(models.Model):
182250
pull_url = models.CharField(max_length=255, null = True, blank = True)
183251
commit_ref = models.CharField(max_length=255, null = True, blank = True)
184252
hash = HashField(null = True, blank = True)
253+
tags = models.ManyToManyField(Tag, through=PatchTag)
254+
255+
objects = PatchManager()
185256

186257
def __unicode__(self):
187258
return self.name
188259

189260
def comments(self):
190261
return Comment.objects.filter(patch = self)
191262

263+
def _set_tag(self, tag, count):
264+
if count == 0:
265+
self.patchtag_set.filter(tag=tag).delete()
266+
return
267+
(patchtag, _) = PatchTag.objects.get_or_create(patch=self, tag=tag)
268+
if patchtag.count != count:
269+
patchtag.count = count
270+
patchtag.save()
271+
272+
def refresh_tag_counts(self):
273+
tags = self.project.tags
274+
counter = Counter()
275+
for comment in self.comment_set.all():
276+
counter = counter + extract_tags(comment.content, tags)
277+
278+
for tag in tags:
279+
self._set_tag(tag, counter[tag])
280+
192281
def save(self):
193282
try:
194283
s = self.state
@@ -239,6 +328,14 @@ def patch_responses(self):
239328
return ''.join([ match.group(0) + '\n' for match in
240329
self.response_re.finditer(self.content)])
241330

331+
def save(self, *args, **kwargs):
332+
super(Comment, self).save(*args, **kwargs)
333+
self.patch.refresh_tag_counts()
334+
335+
def delete(self, *args, **kwargs):
336+
super(Comment, self).delete(*args, **kwargs)
337+
self.patch.refresh_tag_counts()
338+
242339
class Meta:
243340
ordering = ['date']
244341
unique_together = [('msgid', 'patch')]

0 commit comments

Comments
 (0)