Skip to content

Commit eb4eea9

Browse files
ktfktf
authored
DPL: improve resource management (#2513)
Rework the way DPL assign resources to devices. This is a first step in moving away from the hardcoded 127.0.0.1 ip address and allows DPL workflows to run distributed without a third party configuring the channels. A further optimization will also allow DPL to configure ipc / shared memory when two devices happen to be assigned to the same machine.
1 parent 8e2ecab commit eb4eea9

20 files changed

+380
-183
lines changed

‎Framework/Core/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ o2_add_library(Framework
3636
src/CommonDataProcessors.cxx
3737
src/CompletionPolicy.cxx
3838
src/CompletionPolicyHelpers.cxx
39+
src/ComputingResourceHelpers.cxx
3940
src/DispatchPolicy.cxx
4041
src/ConfigParamsHelper.cxx
4142
src/DDSConfigHelpers.cxx

‎Framework/Core/include/Framework/ChannelSpec.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ struct InputChannelSpec {
4040
std::string name;
4141
enum ChannelType type;
4242
enum ChannelMethod method;
43+
std::string hostname;
4344
unsigned short port;
4445
};
4546

@@ -52,6 +53,7 @@ struct OutputChannelSpec {
5253
std::string name;
5354
enum ChannelType type;
5455
enum ChannelMethod method;
56+
std::string hostname;
5557
unsigned short port;
5658
size_t listeners;
5759
};

‎Framework/Core/include/Framework/ComputingResource.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
#ifndef O2_FRAMEWORK_COMPUTINGRESOURCE_H_
11+
#define O2_FRAMEWORK_COMPUTINGRESOURCE_H_
12+
13+
#include <string>
14+
15+
namespace o2::framework
16+
{
17+
18+
struct ComputingOffer {
19+
float cpu = 0;
20+
float memory = 0;
21+
std::string hostname = "";
22+
unsigned short startPort = 0;
23+
unsigned short rangeSize = 0;
24+
};
25+
26+
/// A computing resource which can be offered to run a device
27+
struct ComputingResource {
28+
ComputingResource() = default;
29+
ComputingResource(ComputingOffer const& offer)
30+
: cpu(offer.cpu),
31+
memory(offer.memory),
32+
hostname(offer.hostname),
33+
startPort(offer.startPort),
34+
lastPort(offer.startPort),
35+
usedPorts(0)
36+
{
37+
}
38+
39+
float cpu = 0;
40+
float memory = 0;
41+
std::string hostname = "";
42+
unsigned short startPort = 0;
43+
unsigned short lastPort = 0;
44+
unsigned short usedPorts = 0;
45+
};
46+
47+
} // namespace o2::framework
48+
49+
#endif // O2_FRAMEWORK_COMPUTINGRESOURCES_H_

‎Framework/Core/include/Framework/DeviceSpec.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#define FRAMEWORK_DEVICESPEC_H
1212

1313
#include "Framework/WorkflowSpec.h"
14+
#include "Framework/ComputingResource.h"
1415
#include "Framework/DataProcessorSpec.h"
1516
#include "Framework/ChannelSpec.h"
1617
#include "Framework/ChannelInfo.h"
@@ -54,6 +55,7 @@ struct DeviceSpec {
5455
/// The completion policy to use for this device.
5556
CompletionPolicy completionPolicy;
5657
DispatchPolicy dispatchPolicy;
58+
ComputingResource resource;
5759
};
5860

5961
} // namespace framework

‎Framework/Core/src/ChannelSpecHelpers.cxx‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@
88
// granted to it by virtue of its status as an Intergovernmental Organization
99
// or submit itself to any jurisdiction.
1010
#include "ChannelSpecHelpers.h"
11+
#include <fmt/format.h>
1112
#include <ostream>
1213
#include <cassert>
1314
#include <stdexcept>
1415

15-
namespace o2
16-
{
17-
namespace framework
16+
namespace o2::framework
1817
{
1918

2019
char const* ChannelSpecHelpers::typeAsString(enum ChannelType type)
@@ -43,9 +42,16 @@ char const* ChannelSpecHelpers::methodAsString(enum ChannelMethod method)
4342
throw std::runtime_error("Unknown ChannelMethod");
4443
}
4544

46-
char const* ChannelSpecHelpers::methodAsUrl(enum ChannelMethod method)
45+
std::string ChannelSpecHelpers::channelUrl(OutputChannelSpec const& channel)
46+
{
47+
return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
48+
: fmt::format("tcp://{}:{}", channel.hostname, channel.port);
49+
}
50+
51+
std::string ChannelSpecHelpers::channelUrl(InputChannelSpec const& channel)
4752
{
48-
return (method == ChannelMethod::Bind ? "tcp://*:%d" : "tcp://127.0.0.1:%d");
53+
return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
54+
: fmt::format("tcp://{}:{}", channel.hostname, channel.port);
4955
}
5056

5157
/// Stream operators so that we can use ChannelType with Boost.Test
@@ -62,5 +68,4 @@ std::ostream& operator<<(std::ostream& s, ChannelMethod const& method)
6268
return s;
6369
}
6470

65-
} // namespace framework
66-
} // namespace o2
71+
} // namespace o2::framework

‎Framework/Core/src/ChannelSpecHelpers.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ struct ChannelSpecHelpers {
2626
static char const* typeAsString(enum ChannelType type);
2727
/// return a ChannelMethod as a lowercase string
2828
static char const* methodAsString(enum ChannelMethod method);
29-
/// return a ChannelMethod as a connection url
30-
/// FIXME: currently it only supports tcp://127.0.0.1 and tcp://*, we should
31-
/// have the actual address customizable.
32-
static char const* methodAsUrl(enum ChannelMethod method);
29+
/// @return a url associated to an InputChannelSpec
30+
static std::string channelUrl(InputChannelSpec const&);
31+
/// @return a url associated to an OutputChannelSpec
32+
static std::string channelUrl(OutputChannelSpec const&);
3333
};
3434

3535
/// Stream operators so that we can use ChannelType with Boost.Test

‎Framework/Core/src/ComputingResourceHelpers.cxx‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
#include "ComputingResourceHelpers.h"
11+
#include <thread>
12+
#include <unistd.h>
13+
14+
namespace o2::framework
15+
{
16+
long getTotalNumberOfBytes()
17+
{
18+
long pages = sysconf(_SC_PHYS_PAGES);
19+
long page_size = sysconf(_SC_PAGE_SIZE);
20+
return pages * page_size;
21+
};
22+
23+
ComputingResource ComputingResourceHelpers::getLocalhostResource(unsigned short startPort, unsigned short rangeSize)
24+
{
25+
ComputingResource result;
26+
result.cpu = std::thread::hardware_concurrency(),
27+
result.memory = getTotalNumberOfBytes();
28+
result.hostname = "localhost";
29+
result.startPort = startPort;
30+
result.lastPort = startPort + rangeSize;
31+
result.usedPorts = 0;
32+
return result;
33+
}
34+
} // namespace o2::framework

Framework/Core/src/ComputingResource.h renamed to Framework/Core/src/ComputingResourceHelpers.h

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,25 +7,17 @@
77
// In applying this license CERN does not waive the privileges and immunities
88
// granted to it by virtue of its status as an Intergovernmental Organization
99
// or submit itself to any jurisdiction.
10-
#ifndef FRAMEWORK_COMPUTINGRESOURCE_H
11-
#define FRAMEWORK_COMPUTINGRESOURCE_H
1210

13-
#include <string>
11+
#ifndef O2_FRAMEWORK_COMPUTINGRESOURCEHELPERS_H_
12+
#define O2_FRAMEWORK_COMPUTINGRESOURCEHELPERS_H_
1413

15-
namespace o2
16-
{
17-
namespace framework
18-
{
14+
#include "Framework/ComputingResource.h"
1915

20-
/// A computing resource which can be used to run a device.
21-
struct ComputingResource {
22-
float cpu;
23-
float memory;
24-
std::string hostname;
25-
unsigned short port;
16+
namespace o2::framework
17+
{
18+
struct ComputingResourceHelpers {
19+
static ComputingResource getLocalhostResource(unsigned short startPort, unsigned short rangeSize);
2620
};
21+
} // namespace o2::framework
2722

28-
} // namespace framework
29-
} // namespace o2
30-
31-
#endif // FRAMEWORK_COMPUTINGRESOURCES_H
23+
#endif // O2_FRAMEWORK_COMPUTINGRESOURCEHELPERS_H_

0 commit comments

Comments
 (0)