Skip to content

Commit 53a017b

Browse files
patrickmannlaura-b-gousmaneo
authored
Add deprecation flag to paginated pipeline API (#24183)
* create collection for pipeline metadata * resolve rules * rule walker * add collection for inputs metadata * handle gl2_source_input * REST resource * handle RulesChangedEvent * refactor collection upsert * API for inputs metadata * remove unneeded synchronization * more event handlers * handle input deletion * add a bulk retrieval endpoint * unit test * missing audit event annotation * unit test with pipeline parsing * paginated deprecation flag * Adds 'Deprecated Function' Label to Pipelines Section (#24018) * Created deprecated function constant and deprecated function info component and added component to the 'Manage Rules' page and the pipeline details page. * Fixed FE test and adjusted deprecated function badge placement and sizing. * Use rule deprecation info properly * Remove empty columns * deprecated functions for rule * Show deprecated function label on rules list * Add changelog * filter deprecated functions by rule * Use rule call for rules list on pipeline details view --------- Co-authored-by: Laura Bergenthal-Grotlüschen <laura.bergenthalgrotlueschen@graylog.com> Co-authored-by: patrickmann <patrick.mann@graylog.com> Co-authored-by: Ousmane SAMBA <ousmane@graylog.com>
1 parent 3d1b684 commit 53a017b

File tree

14 files changed

+330
-9
lines changed

14 files changed

+330
-9
lines changed

‎changelog/unreleased/pr-24018.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "Added" # One of: a(dded), c(hanged), d(eprecated), r(emoved), f(ixed), s(ecurity)
2+
message = "Adds a deprecated label to rules and pipelines which shows deprecated functions used in those."
3+
4+
issues = ["23924"]
5+
pulls = ["24018"]

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineRulesMetadataDao.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.graylog.plugins.pipelineprocessor.db;
1818

19+
import com.fasterxml.jackson.annotation.JsonIgnore;
1920
import com.fasterxml.jackson.annotation.JsonProperty;
2021
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2122
import com.google.auto.value.AutoValue;
@@ -52,6 +53,11 @@ public abstract class PipelineRulesMetadataDao implements BuildableMongoEntity<P
5253
@JsonProperty(FIELD_HAS_INPUT_REFERENCES)
5354
public abstract Boolean hasInputReferences();
5455

56+
@JsonIgnore
57+
public boolean hasDeprecatedFunctions() {
58+
return deprecatedFunctions() != null ? !deprecatedFunctions().isEmpty() : false;
59+
}
60+
5561
public static Builder builder() {
5662
return new AutoValue_PipelineRulesMetadataDao.Builder()
5763
.pipelineId("")

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineMetadataService.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,11 @@
3131
import org.slf4j.Logger;
3232
import org.slf4j.LoggerFactory;
3333

34+
import java.util.ArrayList;
3435
import java.util.Collection;
3536
import java.util.HashSet;
3637
import java.util.List;
38+
import java.util.Map;
3739
import java.util.Set;
3840
import java.util.stream.Collectors;
3941

@@ -66,6 +68,13 @@ public PipelineRulesMetadataDao get(final String pipelineId) throws NotFoundExce
6668
return dao;
6769
}
6870

71+
public Map<String, PipelineRulesMetadataDao> get(Set<String> pipelineIds) {
72+
return collection.find(Filters.in(PipelineRulesMetadataDao.FIELD_PIPELINE_ID, pipelineIds))
73+
.into(new ArrayList<>())
74+
.stream()
75+
.collect(Collectors.toMap(PipelineRulesMetadataDao::pipelineId, dao -> dao));
76+
}
77+
6978
public Set<String> getPipelinesByRule(final String ruleId) {
7079
return collection.find(eq(PipelineRulesMetadataDao.FIELD_RULES, ruleId))
7180
.map(PipelineRulesMetadataDao::pipelineId)
@@ -109,4 +118,13 @@ public void delete(Collection<String> pipelineIds) {
109118
LOG.debug("Deleted {} pipeline rules metadata records.", deleteResult.getDeletedCount());
110119
}
111120
}
121+
122+
public Set<String> deprecatedFunctionsPipeline(String pipelineId) {
123+
try {
124+
final PipelineRulesMetadataDao dao = get(pipelineId);
125+
return dao.deprecatedFunctions();
126+
} catch (NotFoundException ignored) {
127+
return Set.of();
128+
}
129+
}
112130
}

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@
7676
import java.util.Collection;
7777
import java.util.HashSet;
7878
import java.util.List;
79+
import java.util.Map;
80+
import java.util.Set;
7981
import java.util.function.Predicate;
8082
import java.util.stream.Collectors;
8183

@@ -234,9 +236,13 @@ public PaginatedResponse<PipelineSource> getPage(@ApiParam(name = "page") @Query
234236

235237
final PaginatedList<PipelineDao> result = paginatedPipelineService
236238
.findPaginated(searchQuery, filter, page, perPage, sort, order);
239+
final Map<String, PipelineRulesMetadataDao> metadataDaos = metadataService.get(
240+
result.stream().map(PipelineDao::id).collect(Collectors.toSet())
241+
);
237242
final List<PipelineSource> pipelineList = result.stream()
238-
.map(dao -> PipelineSource.fromDao(pipelineRuleParser, dao))
239-
.collect(Collectors.toList());
243+
.map(dao -> PipelineSource.fromDao(
244+
pipelineRuleParser, dao, metadataDaos.get(dao.id()) != null && metadataDaos.get(dao.id()).hasDeprecatedFunctions()))
245+
.toList();
240246
final PaginatedList<PipelineSource> pipelines = new PaginatedList<>(pipelineList,
241247
result.pagination().total(), result.pagination().page(), result.pagination().perPage());
242248
return PaginatedResponse.create("pipelines", pipelines);
@@ -259,6 +265,15 @@ public PipelineRulesMetadataDao getRulesMetadata(@ApiParam(name = "id") @PathPar
259265
return metadataService.get(id);
260266
}
261267

268+
@ApiOperation(value = "Get list of deprecated functions used in specified rule")
269+
@Path("/rule/{id}/deprecated_functions")
270+
@GET
271+
public Set<String> getDeprecatedFunctionsForRule(@ApiParam(name = "id") @PathParam("id") String id)
272+
throws NotFoundException {
273+
checkPermission(PipelineRestPermissions.PIPELINE_RULE_READ, id);
274+
return deprecatedFunctionsRule(id);
275+
}
276+
262277
@ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied")
263278
@Path("/{id}")
264279
@PUT
@@ -301,7 +316,7 @@ public RoutingResponse routing(@ApiParam(name = "body", required = true) @NotNul
301316

302317
// Add rule to existing pipeline
303318
PipelineSource pipelineSource = PipelineSource.fromDao(pipelineRuleParser, pipelineDao);
304-
final List<String> rules0 = pipelineSource.stages().get(0).rules();
319+
final List<String> rules0 = pipelineSource.stages().getFirst().rules();
305320
if (rules0.stream().filter(ruleRef -> ruleRef.equals(ruleDao.title())).findFirst().isEmpty()) {
306321
rules0.add(ruleDao.title());
307322
pipelineSource = pipelineSource.toBuilder()
@@ -370,4 +385,12 @@ private void checkSystemRuleUsed(@NotNull PipelineSource pipelineSource) {
370385
}
371386

372387
}
388+
389+
private Set<String> deprecatedFunctionsRule(String ruleId) throws NotFoundException {
390+
Set<String> superset = metadataService.getPipelinesByRule(ruleId).stream()
391+
.flatMap(pipelineId -> metadataService.deprecatedFunctionsPipeline(pipelineId).stream())
392+
.collect(Collectors.toSet());
393+
final RuleDao rule = ruleService.load(ruleId);
394+
return superset.stream().filter(func -> rule.source().contains(func)).collect(Collectors.toSet());
395+
}
373396
}

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ public abstract class PipelineSource {
7575
@Nullable
7676
public abstract Set<ParseError> errors();
7777

78+
@JsonProperty("has_deprecated_functions")
79+
@Nullable
80+
public abstract Boolean hasDeprecatedFunctions();
81+
7882
public static Builder builder() {
7983
return new AutoValue_PipelineSource.Builder();
8084
}
@@ -99,10 +103,15 @@ public static PipelineSource create(@Nullable @JsonProperty("id") @Id @ObjectId
99103
.createdAt(createdAt)
100104
.modifiedAt(modifiedAt)
101105
.stages(stages == null ? Collections.emptyList() : stages)
106+
.hasDeprecatedFunctions(null)
102107
.build();
103108
}
104109

105110
public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) {
111+
return fromDao(parser, dao, null);
112+
}
113+
114+
public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao, Boolean hasDeprecatedFunctions) {
106115
Set<ParseError> errors = null;
107116
Pipeline pipeline = null;
108117
try {
@@ -129,6 +138,7 @@ public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao)
129138
.modifiedAt(dao.modifiedAt())
130139
.stages(stageSources)
131140
.errors(errors)
141+
.hasDeprecatedFunctions(hasDeprecatedFunctions)
132142
.build();
133143
}
134144

@@ -153,5 +163,7 @@ public abstract static class Builder {
153163
public abstract Builder stages(List<StageSource> stages);
154164

155165
public abstract Builder errors(Set<ParseError> errors);
166+
167+
public abstract Builder hasDeprecatedFunctions(Boolean hasDeprecatedFunctions);
156168
}
157169
}

‎graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.StartInput.test.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,7 @@ describe('InputSetupWizard Start Input', () => {
401401
modified_at: '',
402402
errors: [],
403403
_scope: undefined,
404+
has_deprecated_functions: false,
404405
}),
405406
);
406407
});

‎graylog2-web-interface/src/components/pipelines/PipelineListItem.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import type { PipelineConnectionsType } from 'stores/pipelines/PipelineConnectio
2727
import type { Stream } from 'logic/streams/types';
2828
import { defaultCompare as naturalSort } from 'logic/DefaultCompare';
2929
import useGetPermissionsByScope from 'hooks/useScopePermissions';
30+
import RuleDeprecationInfo from 'components/rules/RuleDeprecationInfo';
3031
import usePermissions from 'hooks/usePermissions';
3132

3233
import ButtonToolbar from '../bootstrap/ButtonToolbar';
@@ -123,6 +124,7 @@ const PipelineListItem = ({ pipeline, pipelines, connections, streams, onDeleteP
123124
Managed by Application
124125
</DefaultLabel>
125126
)}
127+
<RuleDeprecationInfo pipelineId={pipeline.id} showFor="pipeline" />
126128
<br />
127129
{description}
128130
<br />

‎graylog2-web-interface/src/components/pipelines/StageRules.tsx‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,14 @@
1515
* <http://www.mongodb.com/licensing/server-side-public-license>.
1616
*/
1717
import React from 'react';
18-
import styled from 'styled-components';
1918

2019
import { DataTable, Icon } from 'components/common';
2120
import { Link } from 'components/common/router';
2221
import Routes from 'routing/Routes';
2322
import { MetricContainer, CounterRate } from 'components/metrics';
2423
import type { PipelineType, StageType } from 'components/pipelines/types';
2524
import type { RuleType } from 'stores/rules/RulesStore';
26-
27-
const TitleTd = styled.td`
28-
width: 400px;
29-
`;
25+
import RuleDeprecationInfo from 'components/rules/RuleDeprecationInfo';
3026

3127
type Props = {
3228
pipeline: PipelineType;
@@ -62,7 +58,10 @@ const StageRules = ({ pipeline, stage, rules = [] }: Props) => {
6258

6359
return (
6460
<tr key={rule.id}>
65-
<TitleTd>{ruleTitle}</TitleTd>
61+
<td>
62+
{ruleTitle}
63+
<RuleDeprecationInfo ruleId={rule.id} />
64+
</td>
6665
<td>{rule.description}</td>
6766
<td>
6867
<MetricContainer

‎graylog2-web-interface/src/components/pipelines/types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export type PipelineType = {
3636
modified_at: string;
3737
stages: Array<StageType>;
3838
errors: Array<ParseError> | null;
39+
has_deprecated_functions: boolean;
3940
_scope: string;
4041
};
4142

@@ -52,4 +53,5 @@ export const DEFAULT_PIPELINE = {
5253
created_at: '',
5354
modified_at: '',
5455
_scope: 'DEFAULT',
56+
has_deprecated_functions: false,
5557
};

‎graylog2-web-interface/src/components/rules/RuleDeprecationInfo.tsx‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
import styled, { css } from 'styled-components';
18+
import * as React from 'react';
19+
20+
import { Label } from 'components/bootstrap';
21+
import { HoverForHelp } from 'components/common';
22+
import usePipelineRulesMetadata from 'components/rules/hooks/usePipelineRulesMetadata';
23+
import useRuleDeprecatedFunctions from 'components/rules/hooks/useRuleDeprecatedFunctions';
24+
25+
type Props = {
26+
pipelineId?: string;
27+
ruleId?: string;
28+
showFor?: 'pipeline' | 'rule';
29+
};
30+
31+
const DeprecatedLabel = styled(Label)(
32+
({ theme }) => css`
33+
display: inline-flex;
34+
margin-left: ${theme.spacings.xs};
35+
vertical-align: inherit;
36+
gap: ${theme.spacings.xxs};
37+
`,
38+
);
39+
40+
const StyledList = styled.ul(
41+
({ theme }) => css`
42+
display: flex;
43+
flex-direction: column;
44+
gap: ${theme.spacings.xxs};
45+
margin-bottom: ${theme.spacings.xs};
46+
margin-left: ${theme.spacings.xs};
47+
list-style-type: none;
48+
`,
49+
);
50+
51+
const RuleDeprecationInfo = ({ pipelineId = undefined, ruleId = undefined, showFor = 'rule' }: Props) => {
52+
const { data: pipelineMetaData, isLoading: isLoadingPipelineMetaData } = usePipelineRulesMetadata(pipelineId, {
53+
enabled: !!pipelineId,
54+
});
55+
56+
const { data: ruleDeprecatedFunctionsData, isLoading: isLoadingRuleDeprecatedFunctions } = useRuleDeprecatedFunctions(
57+
ruleId,
58+
{
59+
enabled: !!ruleId,
60+
},
61+
);
62+
63+
if (
64+
isLoadingPipelineMetaData ||
65+
isLoadingRuleDeprecatedFunctions ||
66+
!pipelineMetaData ||
67+
!ruleDeprecatedFunctionsData
68+
)
69+
return null;
70+
71+
const getDeprecatedFunctions = () => {
72+
if (pipelineMetaData?.deprecated_functions?.length > 0) {
73+
return pipelineMetaData.deprecated_functions;
74+
}
75+
76+
if (ruleDeprecatedFunctionsData?.length > 0) {
77+
return ruleDeprecatedFunctionsData;
78+
}
79+
80+
return [];
81+
};
82+
83+
const deprecatedFunctions = getDeprecatedFunctions();
84+
85+
if (deprecatedFunctions.length === 0) return null;
86+
87+
return (
88+
<DeprecatedLabel bsStyle="warning">
89+
<span>Deprecated Function</span>
90+
<HoverForHelp trigger="hover" type="info" title={`This ${showFor} contains at least one deprecated function:`}>
91+
<StyledList>
92+
{deprecatedFunctions?.map((func) => (
93+
<li key={func}>{func}</li>
94+
))}
95+
</StyledList>
96+
</HoverForHelp>
97+
</DeprecatedLabel>
98+
);
99+
};
100+
101+
export default RuleDeprecationInfo;

0 commit comments

Comments
 (0)