Skip to content

Commit 7b4725c

Browse files
todvoratodvora
authored
Add support for azure blob storage to datanode (#24567)
* Add support for azure blob storage to datanode * Added changelog * fixed hdfs plugin for aarch64
1 parent e6d6440 commit 7b4725c

File tree

15 files changed

+464
-188
lines changed

15 files changed

+464
-188
lines changed

‎changelog/unreleased/pr-24567.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "a"
2+
message = "Add support for Azure blob storage in datanode"
3+
4+
issues = ["24551"]
5+
pulls = ["24567"]

‎data-node/pom.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,23 @@
634634
</sha512>
635635
</configuration>
636636
</execution>
637+
<execution>
638+
<id>download-repository-azure-plugin</id>
639+
<phase>generate-resources</phase>
640+
<goals>
641+
<goal>wget</goal>
642+
</goals>
643+
<configuration>
644+
<url>
645+
https://artifacts.opensearch.org/releases/plugins/repository-azure/${opensearch.version}/repository-azure-${opensearch.version}.zip
646+
</url>
647+
<outputFileName>repository-azure-${opensearch.version}.zip</outputFileName>
648+
<outputDirectory>${project.build.directory}/opensearch-plugins</outputDirectory>
649+
<sha512>
650+
4f34367bb0949112f0bc73e3cbe84559fe4125f1ba356a219eab37ffe6e3bc634b8176bb2eadc2b3f23c334d68e39269c2be8d9bb442422cda35a7cd8c5c644e
651+
</sha512>
652+
</configuration>
653+
</execution>
637654
</executions>
638655
</plugin>
639656

@@ -735,6 +752,7 @@
735752
<argument>${project.build.directory}/opensearch-plugins/repository-s3-${opensearch.version}.zip</argument>
736753
<argument>${project.build.directory}/opensearch-plugins/repository-gcs-${opensearch.version}.zip</argument>
737754
<argument>${project.build.directory}/opensearch-plugins/repository-hdfs-${opensearch.version}.zip</argument>
755+
<argument>${project.build.directory}/opensearch-plugins/repository-azure-${opensearch.version}.zip</argument>
738756
</arguments>
739757
</configuration>
740758
</execution>
@@ -751,6 +769,7 @@
751769
<argument>${project.build.directory}/opensearch-plugins/repository-s3-${opensearch.version}.zip</argument>
752770
<argument>${project.build.directory}/opensearch-plugins/repository-gcs-${opensearch.version}.zip</argument>
753771
<argument>${project.build.directory}/opensearch-plugins/repository-hdfs-${opensearch.version}.zip</argument>
772+
<argument>${project.build.directory}/opensearch-plugins/repository-azure-${opensearch.version}.zip</argument>
754773
</arguments>
755774
</configuration>
756775
</execution>

‎data-node/src/main/java/org/graylog/datanode/Configuration.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -333,13 +333,6 @@ and it is available on another network interface than $http_bind_address,
333333
@Parameter(value = "node_search_cache_size")
334334
private String searchCacheSize = "10gb";
335335

336-
/**
337-
* <a href="https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/#shared-file-system">See snapshot documentation</a>
338-
*/
339-
@Documentation("Filesystem path where searchable snapshots should be stored")
340-
@Parameter(value = "path_repo", converter = PathListConverter.class, validators = DirectoriesWritableValidator.class)
341-
private List<Path> pathRepo;
342-
343336
@Documentation("This setting limits the number of clauses a Lucene BooleanQuery can have.")
344337
@Parameter(value = "opensearch_indices_query_bool_max_clause_count")
345338
private Integer indicesQueryBoolMaxClauseCount = 32768;
@@ -715,10 +708,6 @@ public String getNodeSearchCacheSize() {
715708
return searchCacheSize;
716709
}
717710

718-
public List<Path> getPathRepo() {
719-
return pathRepo;
720-
}
721-
722711
public List<String> getNodeRoles() {
723712
return nodeRoles;
724713
}

‎data-node/src/main/java/org/graylog/datanode/bindings/SearchableSnapshotsBindings.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
package org.graylog.datanode.bindings;
18+
19+
import com.google.inject.AbstractModule;
20+
import com.google.inject.multibindings.MapBinder;
21+
import com.google.inject.multibindings.Multibinder;
22+
import org.graylog.datanode.configuration.snapshots.RepositoryConfiguration;
23+
24+
import java.util.Set;
25+
26+
/**
27+
* The sole purpose of this module is to provide repositories as Set<RepositoryConfiguration> for injection.
28+
*/
29+
public class SearchableSnapshotsBindings extends AbstractModule {
30+
private final Set<RepositoryConfiguration> repositories;
31+
32+
public SearchableSnapshotsBindings(Set<RepositoryConfiguration> repositories) {
33+
this.repositories = repositories;
34+
}
35+
36+
@Override
37+
protected void configure() {
38+
final Multibinder<RepositoryConfiguration> multibinder = Multibinder.newSetBinder(binder(), RepositoryConfiguration.class);
39+
repositories.forEach(r -> multibinder.addBinding().toInstance(r));
40+
}
41+
}

‎data-node/src/main/java/org/graylog/datanode/commands/Datanode.java‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,16 @@
3131
import org.graylog.datanode.bindings.ConfigurationModule;
3232
import org.graylog.datanode.bindings.DatanodeServerBindings;
3333
import org.graylog.datanode.bindings.PeriodicalBindings;
34+
import org.graylog.datanode.bindings.SearchableSnapshotsBindings;
3435
import org.graylog.datanode.bootstrap.DatanodeBootstrap;
3536
import org.graylog.datanode.bootstrap.Main;
37+
import org.graylog.datanode.configuration.snapshots.AzureRepositoryConfiguration;
3638
import org.graylog.datanode.configuration.DatanodeProvisioningBindings;
37-
import org.graylog.datanode.configuration.GCSRepositoryConfiguration;
38-
import org.graylog.datanode.configuration.HdfsRepositoryConfiguration;
39-
import org.graylog.datanode.configuration.S3RepositoryConfiguration;
39+
import org.graylog.datanode.configuration.snapshots.FsRepositoryConfiguration;
40+
import org.graylog.datanode.configuration.snapshots.GCSRepositoryConfiguration;
41+
import org.graylog.datanode.configuration.snapshots.HdfsRepositoryConfiguration;
42+
import org.graylog.datanode.configuration.snapshots.RepositoryConfiguration;
43+
import org.graylog.datanode.configuration.snapshots.S3RepositoryConfiguration;
4044
import org.graylog.datanode.rest.RestBindings;
4145
import org.graylog.datanode.shutdown.GracefulShutdown;
4246
import org.graylog2.cluster.nodes.DataNodeDto;
@@ -52,19 +56,25 @@
5256
import org.slf4j.Logger;
5357
import org.slf4j.LoggerFactory;
5458

55-
import java.util.Arrays;
59+
import java.util.ArrayList;
5660
import java.util.Collection;
5761
import java.util.List;
62+
import java.util.Set;
5863

5964

6065
@Command(name = "datanode", description = "Start Graylog Data Node")
6166
public class Datanode extends DatanodeBootstrap implements DocumentedBeansService {
6267
private static final Logger LOG = LoggerFactory.getLogger(Datanode.class);
6368

64-
private final S3RepositoryConfiguration s3RepositoryConfiguration = new S3RepositoryConfiguration();
6569
private final TLSProtocolsConfiguration tlsConfiguration = new TLSProtocolsConfiguration();
66-
private final GCSRepositoryConfiguration gcsRepositoryConfiguration = new GCSRepositoryConfiguration();
67-
private final HdfsRepositoryConfiguration hdfsRepositoryConfiguration = new HdfsRepositoryConfiguration();
70+
71+
private final Set<RepositoryConfiguration> snapshotRepositories = Set.of(
72+
new FsRepositoryConfiguration(),
73+
new S3RepositoryConfiguration(),
74+
new GCSRepositoryConfiguration(),
75+
new HdfsRepositoryConfiguration(),
76+
new AzureRepositoryConfiguration()
77+
);
6878

6979
public Datanode() {
7080
super("datanode", new Configuration());
@@ -78,18 +88,17 @@ public Datanode() {
7888
new DatanodeServerBindings(),
7989
new RestBindings(),
8090
new DatanodeProvisioningBindings(),
81-
new PeriodicalBindings()
91+
new PeriodicalBindings(),
92+
new SearchableSnapshotsBindings(snapshotRepositories)
8293
);
8394
return modules.build();
8495
}
8596

8697
@Override
8798
public @Nonnull List<Object> getNodeCommandConfigurationBeans() {
88-
return Arrays.asList(
89-
tlsConfiguration,
90-
s3RepositoryConfiguration,
91-
gcsRepositoryConfiguration,
92-
hdfsRepositoryConfiguration);
99+
final ArrayList<Object> configurationBeans = new ArrayList<>(snapshotRepositories);
100+
configurationBeans.add(tlsConfiguration);
101+
return configurationBeans;
93102
}
94103

95104
@Override

‎data-node/src/main/java/org/graylog/datanode/configuration/GCSRepositoryConfiguration.java‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎data-node/src/main/java/org/graylog/datanode/configuration/snapshots/AzureRepositoryConfiguration.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
package org.graylog.datanode.configuration.snapshots;
18+
19+
import com.github.joschi.jadconfig.Parameter;
20+
import com.github.joschi.jadconfig.documentation.Documentation;
21+
import com.github.joschi.jadconfig.documentation.DocumentationSection;
22+
import com.google.common.collect.ImmutableList;
23+
import org.apache.commons.lang3.StringUtils;
24+
import org.graylog.datanode.configuration.DatanodeDirectories;
25+
import org.graylog.datanode.process.configuration.beans.OpensearchKeystoreItem;
26+
import org.graylog.datanode.process.configuration.beans.OpensearchKeystoreStringItem;
27+
28+
import java.util.Collection;
29+
import java.util.Collections;
30+
import java.util.Map;
31+
32+
@DocumentationSection(heading = "Azure repository configuration for searchable snapshots", description = "")
33+
public class AzureRepositoryConfiguration implements RepositoryConfiguration {
34+
35+
@Documentation("Azure client default account for searchable snapshots")
36+
@Parameter(value = "azure_client_default_account")
37+
private String azureClientDefaultAccount;
38+
39+
@Documentation("Azure client default account for searchable snapshots")
40+
@Parameter(value = "azure_client_default_key")
41+
private String azureClientDefaultKey;
42+
43+
@Documentation("Azure client default account for searchable snapshots")
44+
@Parameter(value = "azure_client_default_sas_token")
45+
private String azureClientDefaultSasToken;
46+
47+
@Override
48+
public boolean isRepositoryEnabled() {
49+
if (noneBlank(azureClientDefaultAccount) && noneBlank(azureClientDefaultSasToken) || noneBlank(azureClientDefaultKey)) {
50+
// All the required properties are set and not blank, s3 repository is enabled
51+
return true;
52+
} else if (allBlank(azureClientDefaultAccount, azureClientDefaultKey, azureClientDefaultSasToken)) {
53+
// all are empty, this means repository is not configured at all
54+
return false;
55+
} else {
56+
// One or two properties are configured, this is an incomplete configuration we can't handle this situation
57+
throw new IllegalStateException("""
58+
Azure Client not configured properly,
59+
Property azure_client_default_account is required, together with either azure_client_default_key or azure_client_default_sas_token
60+
, they have to be provided in the configuration!""");
61+
}
62+
}
63+
64+
@Override
65+
public Map<String, String> opensearchProperties() {
66+
return Collections.emptyMap();
67+
}
68+
69+
@Override
70+
public Collection<OpensearchKeystoreItem> keystoreItems(DatanodeDirectories datanodeDirectories) {
71+
final ImmutableList.Builder<OpensearchKeystoreItem> builder = ImmutableList.builder();
72+
builder.add(new OpensearchKeystoreStringItem("azure.client.default.account", azureClientDefaultAccount));
73+
if (!StringUtils.isEmpty(azureClientDefaultKey)) {
74+
builder.add(new OpensearchKeystoreStringItem("azure.client.default.key", azureClientDefaultKey));
75+
} else if (!StringUtils.isEmpty(azureClientDefaultSasToken)) {
76+
builder.add(new OpensearchKeystoreStringItem("azure.client.default.sas_token", azureClientDefaultSasToken));
77+
}
78+
return builder.build();
79+
}
80+
}

‎data-node/src/main/java/org/graylog/datanode/configuration/snapshots/FsRepositoryConfiguration.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
package org.graylog.datanode.configuration.snapshots;
18+
19+
import com.github.joschi.jadconfig.Parameter;
20+
import com.github.joschi.jadconfig.documentation.Documentation;
21+
import jakarta.annotation.Nonnull;
22+
import org.graylog.datanode.DirectoriesWritableValidator;
23+
import org.graylog.datanode.PathListConverter;
24+
import org.graylog.datanode.configuration.DatanodeDirectories;
25+
import org.graylog.datanode.process.configuration.beans.OpensearchKeystoreItem;
26+
27+
import java.nio.file.Path;
28+
import java.util.Collection;
29+
import java.util.Collections;
30+
import java.util.List;
31+
import java.util.Map;
32+
import java.util.stream.Collectors;
33+
34+
public class FsRepositoryConfiguration implements RepositoryConfiguration {
35+
/**
36+
* <a href="https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/#shared-file-system">See snapshot documentation</a>
37+
*/
38+
@Documentation("Filesystem path where searchable snapshots should be stored")
39+
@Parameter(value = "path_repo", converter = PathListConverter.class, validators = DirectoriesWritableValidator.class)
40+
private List<Path> pathRepo;
41+
42+
@Override
43+
public boolean isRepositoryEnabled() throws IllegalStateException {
44+
return pathRepo != null && !pathRepo.isEmpty();
45+
}
46+
47+
@Override
48+
public Map<String, String> opensearchProperties() {
49+
// https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/#shared-file-system
50+
return Map.of("path.repo", serialize(pathRepo));
51+
}
52+
53+
@Override
54+
public Collection<OpensearchKeystoreItem> keystoreItems(DatanodeDirectories datanodeDirectories) {
55+
return Collections.emptyList();
56+
}
57+
58+
@Nonnull
59+
private String serialize(List<Path> pathRepo) {
60+
return pathRepo.stream().map(Path::toString).collect(Collectors.joining(","));
61+
}
62+
}

0 commit comments

Comments
 (0)