Skip to content

Commit 482cfc3

Browse files
Florian Westphalummakynes
Florian Westphal
authored and
ummakynes
committed
netfilter: xtables: avoid percpu ruleset duplication
We store the rule blob per (possible) cpu. Unfortunately this means we can waste lot of memory on big smp machines. ipt_entry structure ('rule head') is 112 byte, so e.g. with maxcpu=64 one single rule eats close to 8k RAM. Since previous patch made counters percpu it appears there is nothing left in the rule blob that needs to be percpu. On my test system (144 possible cpus, 400k dummy rules) this change saves close to 9 Gigabyte of RAM. Reported-by: Marcelo Ricardo Leitner <marcelo.leitner@gmail.com> Acked-by: Jesper Dangaard Brouer <brouer@redhat.com> Signed-off-by: Florian Westphal <fw@strlen.de> Acked-by: Eric Dumazet <edumazet@google.com> Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
1 parent 71ae0df commit 482cfc3

5 files changed

Lines changed: 57 additions & 149 deletions

File tree

‎include/linux/netfilter/x_tables.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,9 +224,9 @@ struct xt_table_info {
224224
unsigned int stacksize;
225225
unsigned int __percpu *stackptr;
226226
void ***jumpstack;
227-
/* ipt_entry tables: one per CPU */
227+
228228
/* Note : this field MUST be the last one, see XT_TABLE_INFO_SZ */
229-
void *entries[1];
229+
void *entries;
230230
};
231231

232232
#define XT_TABLE_INFO_SZ (offsetof(struct xt_table_info, entries) \

‎net/ipv4/netfilter/arp_tables.c‎

Lines changed: 15 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ unsigned int arpt_do_table(struct sk_buff *skb,
275275
* pointer.
276276
*/
277277
smp_read_barrier_depends();
278-
table_base = private->entries[smp_processor_id()];
278+
table_base = private->entries;
279279

280280
e = get_entry(table_base, private->hook_entry[hook]);
281281
back = get_entry(table_base, private->underflow[hook]);
@@ -711,12 +711,6 @@ static int translate_table(struct xt_table_info *newinfo, void *entry0,
711711
return ret;
712712
}
713713

714-
/* And one copy for every other CPU */
715-
for_each_possible_cpu(i) {
716-
if (newinfo->entries[i] && newinfo->entries[i] != entry0)
717-
memcpy(newinfo->entries[i], entry0, newinfo->size);
718-
}
719-
720714
return ret;
721715
}
722716

@@ -731,7 +725,7 @@ static void get_counters(const struct xt_table_info *t,
731725
seqcount_t *s = &per_cpu(xt_recseq, cpu);
732726

733727
i = 0;
734-
xt_entry_foreach(iter, t->entries[cpu], t->size) {
728+
xt_entry_foreach(iter, t->entries, t->size) {
735729
struct xt_counters *tmp;
736730
u64 bcnt, pcnt;
737731
unsigned int start;
@@ -785,7 +779,7 @@ static int copy_entries_to_user(unsigned int total_size,
785779
if (IS_ERR(counters))
786780
return PTR_ERR(counters);
787781

788-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
782+
loc_cpu_entry = private->entries;
789783
/* ... then copy entire thing ... */
790784
if (copy_to_user(userptr, loc_cpu_entry, total_size) != 0) {
791785
ret = -EFAULT;
@@ -880,10 +874,10 @@ static int compat_table_info(const struct xt_table_info *info,
880874
if (!newinfo || !info)
881875
return -EINVAL;
882876

883-
/* we dont care about newinfo->entries[] */
877+
/* we dont care about newinfo->entries */
884878
memcpy(newinfo, info, offsetof(struct xt_table_info, entries));
885879
newinfo->initial_entries = 0;
886-
loc_cpu_entry = info->entries[raw_smp_processor_id()];
880+
loc_cpu_entry = info->entries;
887881
xt_compat_init_offsets(NFPROTO_ARP, info->number);
888882
xt_entry_foreach(iter, loc_cpu_entry, info->size) {
889883
ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo);
@@ -1048,7 +1042,7 @@ static int __do_replace(struct net *net, const char *name,
10481042
get_counters(oldinfo, counters);
10491043

10501044
/* Decrease module usage counts and free resource */
1051-
loc_cpu_old_entry = oldinfo->entries[raw_smp_processor_id()];
1045+
loc_cpu_old_entry = oldinfo->entries;
10521046
xt_entry_foreach(iter, loc_cpu_old_entry, oldinfo->size)
10531047
cleanup_entry(iter);
10541048

@@ -1095,8 +1089,7 @@ static int do_replace(struct net *net, const void __user *user,
10951089
if (!newinfo)
10961090
return -ENOMEM;
10971091

1098-
/* choose the copy that is on our node/cpu */
1099-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
1092+
loc_cpu_entry = newinfo->entries;
11001093
if (copy_from_user(loc_cpu_entry, user + sizeof(tmp),
11011094
tmp.size) != 0) {
11021095
ret = -EFAULT;
@@ -1126,7 +1119,7 @@ static int do_replace(struct net *net, const void __user *user,
11261119
static int do_add_counters(struct net *net, const void __user *user,
11271120
unsigned int len, int compat)
11281121
{
1129-
unsigned int i, curcpu;
1122+
unsigned int i;
11301123
struct xt_counters_info tmp;
11311124
struct xt_counters *paddc;
11321125
unsigned int num_counters;
@@ -1136,7 +1129,6 @@ static int do_add_counters(struct net *net, const void __user *user,
11361129
struct xt_table *t;
11371130
const struct xt_table_info *private;
11381131
int ret = 0;
1139-
void *loc_cpu_entry;
11401132
struct arpt_entry *iter;
11411133
unsigned int addend;
11421134
#ifdef CONFIG_COMPAT
@@ -1192,11 +1184,9 @@ static int do_add_counters(struct net *net, const void __user *user,
11921184
}
11931185

11941186
i = 0;
1195-
/* Choose the copy that is on our node */
1196-
curcpu = smp_processor_id();
1197-
loc_cpu_entry = private->entries[curcpu];
1187+
11981188
addend = xt_write_recseq_begin();
1199-
xt_entry_foreach(iter, loc_cpu_entry, private->size) {
1189+
xt_entry_foreach(iter, private->entries, private->size) {
12001190
struct xt_counters *tmp;
12011191

12021192
tmp = xt_get_this_cpu_counter(&iter->counters);
@@ -1410,7 +1400,7 @@ static int translate_compat_table(const char *name,
14101400
newinfo->hook_entry[i] = info->hook_entry[i];
14111401
newinfo->underflow[i] = info->underflow[i];
14121402
}
1413-
entry1 = newinfo->entries[raw_smp_processor_id()];
1403+
entry1 = newinfo->entries;
14141404
pos = entry1;
14151405
size = total_size;
14161406
xt_entry_foreach(iter0, entry0, total_size) {
@@ -1470,11 +1460,6 @@ static int translate_compat_table(const char *name,
14701460
return ret;
14711461
}
14721462

1473-
/* And one copy for every other CPU */
1474-
for_each_possible_cpu(i)
1475-
if (newinfo->entries[i] && newinfo->entries[i] != entry1)
1476-
memcpy(newinfo->entries[i], entry1, newinfo->size);
1477-
14781463
*pinfo = newinfo;
14791464
*pentry0 = entry1;
14801465
xt_free_table_info(info);
@@ -1533,8 +1518,7 @@ static int compat_do_replace(struct net *net, void __user *user,
15331518
if (!newinfo)
15341519
return -ENOMEM;
15351520

1536-
/* choose the copy that is on our node/cpu */
1537-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
1521+
loc_cpu_entry = newinfo->entries;
15381522
if (copy_from_user(loc_cpu_entry, user + sizeof(tmp), tmp.size) != 0) {
15391523
ret = -EFAULT;
15401524
goto free_newinfo;
@@ -1631,19 +1615,16 @@ static int compat_copy_entries_to_user(unsigned int total_size,
16311615
void __user *pos;
16321616
unsigned int size;
16331617
int ret = 0;
1634-
void *loc_cpu_entry;
16351618
unsigned int i = 0;
16361619
struct arpt_entry *iter;
16371620

16381621
counters = alloc_counters(table);
16391622
if (IS_ERR(counters))
16401623
return PTR_ERR(counters);
16411624

1642-
/* choose the copy on our node/cpu */
1643-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
16441625
pos = userptr;
16451626
size = total_size;
1646-
xt_entry_foreach(iter, loc_cpu_entry, total_size) {
1627+
xt_entry_foreach(iter, private->entries, total_size) {
16471628
ret = compat_copy_entry_to_user(iter, &pos,
16481629
&size, counters, i++);
16491630
if (ret != 0)
@@ -1812,8 +1793,7 @@ struct xt_table *arpt_register_table(struct net *net,
18121793
goto out;
18131794
}
18141795

1815-
/* choose the copy on our node/cpu */
1816-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
1796+
loc_cpu_entry = newinfo->entries;
18171797
memcpy(loc_cpu_entry, repl->entries, repl->size);
18181798

18191799
ret = translate_table(newinfo, loc_cpu_entry, repl);
@@ -1844,7 +1824,7 @@ void arpt_unregister_table(struct xt_table *table)
18441824
private = xt_unregister_table(table);
18451825

18461826
/* Decrease module usage counts and free resources */
1847-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
1827+
loc_cpu_entry = private->entries;
18481828
xt_entry_foreach(iter, loc_cpu_entry, private->size)
18491829
cleanup_entry(iter);
18501830
if (private->number > private->initial_entries)

‎net/ipv4/netfilter/ip_tables.c‎

Lines changed: 16 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -254,15 +254,13 @@ static void trace_packet(const struct sk_buff *skb,
254254
const struct xt_table_info *private,
255255
const struct ipt_entry *e)
256256
{
257-
const void *table_base;
258257
const struct ipt_entry *root;
259258
const char *hookname, *chainname, *comment;
260259
const struct ipt_entry *iter;
261260
unsigned int rulenum = 0;
262261
struct net *net = dev_net(in ? in : out);
263262

264-
table_base = private->entries[smp_processor_id()];
265-
root = get_entry(table_base, private->hook_entry[hook]);
263+
root = get_entry(private->entries, private->hook_entry[hook]);
266264

267265
hookname = chainname = hooknames[hook];
268266
comment = comments[NF_IP_TRACE_COMMENT_RULE];
@@ -331,7 +329,7 @@ ipt_do_table(struct sk_buff *skb,
331329
* pointer.
332330
*/
333331
smp_read_barrier_depends();
334-
table_base = private->entries[cpu];
332+
table_base = private->entries;
335333
jumpstack = (struct ipt_entry **)private->jumpstack[cpu];
336334
stackptr = per_cpu_ptr(private->stackptr, cpu);
337335
origptr = *stackptr;
@@ -877,12 +875,6 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
877875
return ret;
878876
}
879877

880-
/* And one copy for every other CPU */
881-
for_each_possible_cpu(i) {
882-
if (newinfo->entries[i] && newinfo->entries[i] != entry0)
883-
memcpy(newinfo->entries[i], entry0, newinfo->size);
884-
}
885-
886878
return ret;
887879
}
888880

@@ -898,7 +890,7 @@ get_counters(const struct xt_table_info *t,
898890
seqcount_t *s = &per_cpu(xt_recseq, cpu);
899891

900892
i = 0;
901-
xt_entry_foreach(iter, t->entries[cpu], t->size) {
893+
xt_entry_foreach(iter, t->entries, t->size) {
902894
struct xt_counters *tmp;
903895
u64 bcnt, pcnt;
904896
unsigned int start;
@@ -946,17 +938,13 @@ copy_entries_to_user(unsigned int total_size,
946938
struct xt_counters *counters;
947939
const struct xt_table_info *private = table->private;
948940
int ret = 0;
949-
const void *loc_cpu_entry;
941+
void *loc_cpu_entry;
950942

951943
counters = alloc_counters(table);
952944
if (IS_ERR(counters))
953945
return PTR_ERR(counters);
954946

955-
/* choose the copy that is on our node/cpu, ...
956-
* This choice is lazy (because current thread is
957-
* allowed to migrate to another cpu)
958-
*/
959-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
947+
loc_cpu_entry = private->entries;
960948
if (copy_to_user(userptr, loc_cpu_entry, total_size) != 0) {
961949
ret = -EFAULT;
962950
goto free_counters;
@@ -1070,10 +1058,10 @@ static int compat_table_info(const struct xt_table_info *info,
10701058
if (!newinfo || !info)
10711059
return -EINVAL;
10721060

1073-
/* we dont care about newinfo->entries[] */
1061+
/* we dont care about newinfo->entries */
10741062
memcpy(newinfo, info, offsetof(struct xt_table_info, entries));
10751063
newinfo->initial_entries = 0;
1076-
loc_cpu_entry = info->entries[raw_smp_processor_id()];
1064+
loc_cpu_entry = info->entries;
10771065
xt_compat_init_offsets(AF_INET, info->number);
10781066
xt_entry_foreach(iter, loc_cpu_entry, info->size) {
10791067
ret = compat_calc_entry(iter, info, loc_cpu_entry, newinfo);
@@ -1194,7 +1182,6 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks,
11941182
struct xt_table *t;
11951183
struct xt_table_info *oldinfo;
11961184
struct xt_counters *counters;
1197-
void *loc_cpu_old_entry;
11981185
struct ipt_entry *iter;
11991186

12001187
ret = 0;
@@ -1237,8 +1224,7 @@ __do_replace(struct net *net, const char *name, unsigned int valid_hooks,
12371224
get_counters(oldinfo, counters);
12381225

12391226
/* Decrease module usage counts and free resource */
1240-
loc_cpu_old_entry = oldinfo->entries[raw_smp_processor_id()];
1241-
xt_entry_foreach(iter, loc_cpu_old_entry, oldinfo->size)
1227+
xt_entry_foreach(iter, oldinfo->entries, oldinfo->size)
12421228
cleanup_entry(iter, net);
12431229

12441230
xt_free_table_info(oldinfo);
@@ -1284,8 +1270,7 @@ do_replace(struct net *net, const void __user *user, unsigned int len)
12841270
if (!newinfo)
12851271
return -ENOMEM;
12861272

1287-
/* choose the copy that is on our node/cpu */
1288-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
1273+
loc_cpu_entry = newinfo->entries;
12891274
if (copy_from_user(loc_cpu_entry, user + sizeof(tmp),
12901275
tmp.size) != 0) {
12911276
ret = -EFAULT;
@@ -1316,7 +1301,7 @@ static int
13161301
do_add_counters(struct net *net, const void __user *user,
13171302
unsigned int len, int compat)
13181303
{
1319-
unsigned int i, curcpu;
1304+
unsigned int i;
13201305
struct xt_counters_info tmp;
13211306
struct xt_counters *paddc;
13221307
unsigned int num_counters;
@@ -1326,7 +1311,6 @@ do_add_counters(struct net *net, const void __user *user,
13261311
struct xt_table *t;
13271312
const struct xt_table_info *private;
13281313
int ret = 0;
1329-
void *loc_cpu_entry;
13301314
struct ipt_entry *iter;
13311315
unsigned int addend;
13321316
#ifdef CONFIG_COMPAT
@@ -1382,11 +1366,8 @@ do_add_counters(struct net *net, const void __user *user,
13821366
}
13831367

13841368
i = 0;
1385-
/* Choose the copy that is on our node */
1386-
curcpu = smp_processor_id();
1387-
loc_cpu_entry = private->entries[curcpu];
13881369
addend = xt_write_recseq_begin();
1389-
xt_entry_foreach(iter, loc_cpu_entry, private->size) {
1370+
xt_entry_foreach(iter, private->entries, private->size) {
13901371
struct xt_counters *tmp;
13911372

13921373
tmp = xt_get_this_cpu_counter(&iter->counters);
@@ -1739,7 +1720,7 @@ translate_compat_table(struct net *net,
17391720
newinfo->hook_entry[i] = info->hook_entry[i];
17401721
newinfo->underflow[i] = info->underflow[i];
17411722
}
1742-
entry1 = newinfo->entries[raw_smp_processor_id()];
1723+
entry1 = newinfo->entries;
17431724
pos = entry1;
17441725
size = total_size;
17451726
xt_entry_foreach(iter0, entry0, total_size) {
@@ -1791,11 +1772,6 @@ translate_compat_table(struct net *net,
17911772
return ret;
17921773
}
17931774

1794-
/* And one copy for every other CPU */
1795-
for_each_possible_cpu(i)
1796-
if (newinfo->entries[i] && newinfo->entries[i] != entry1)
1797-
memcpy(newinfo->entries[i], entry1, newinfo->size);
1798-
17991775
*pinfo = newinfo;
18001776
*pentry0 = entry1;
18011777
xt_free_table_info(info);
@@ -1842,8 +1818,7 @@ compat_do_replace(struct net *net, void __user *user, unsigned int len)
18421818
if (!newinfo)
18431819
return -ENOMEM;
18441820

1845-
/* choose the copy that is on our node/cpu */
1846-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
1821+
loc_cpu_entry = newinfo->entries;
18471822
if (copy_from_user(loc_cpu_entry, user + sizeof(tmp),
18481823
tmp.size) != 0) {
18491824
ret = -EFAULT;
@@ -1914,22 +1889,16 @@ compat_copy_entries_to_user(unsigned int total_size, struct xt_table *table,
19141889
void __user *pos;
19151890
unsigned int size;
19161891
int ret = 0;
1917-
const void *loc_cpu_entry;
19181892
unsigned int i = 0;
19191893
struct ipt_entry *iter;
19201894

19211895
counters = alloc_counters(table);
19221896
if (IS_ERR(counters))
19231897
return PTR_ERR(counters);
19241898

1925-
/* choose the copy that is on our node/cpu, ...
1926-
* This choice is lazy (because current thread is
1927-
* allowed to migrate to another cpu)
1928-
*/
1929-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
19301899
pos = userptr;
19311900
size = total_size;
1932-
xt_entry_foreach(iter, loc_cpu_entry, total_size) {
1901+
xt_entry_foreach(iter, private->entries, total_size) {
19331902
ret = compat_copy_entry_to_user(iter, &pos,
19341903
&size, counters, i++);
19351904
if (ret != 0)
@@ -2104,8 +2073,7 @@ struct xt_table *ipt_register_table(struct net *net,
21042073
goto out;
21052074
}
21062075

2107-
/* choose the copy on our node/cpu, but dont care about preemption */
2108-
loc_cpu_entry = newinfo->entries[raw_smp_processor_id()];
2076+
loc_cpu_entry = newinfo->entries;
21092077
memcpy(loc_cpu_entry, repl->entries, repl->size);
21102078

21112079
ret = translate_table(net, newinfo, loc_cpu_entry, repl);
@@ -2136,7 +2104,7 @@ void ipt_unregister_table(struct net *net, struct xt_table *table)
21362104
private = xt_unregister_table(table);
21372105

21382106
/* Decrease module usage counts and free resources */
2139-
loc_cpu_entry = private->entries[raw_smp_processor_id()];
2107+
loc_cpu_entry = private->entries;
21402108
xt_entry_foreach(iter, loc_cpu_entry, private->size)
21412109
cleanup_entry(iter, net);
21422110
if (private->number > private->initial_entries)

0 commit comments

Comments
 (0)