Skip to content

Commit bf5f85c

Browse files
giltirngiltirn
committed
Merge branch 'ckelly_develop'
2 parents 6be5578 + f1a7838 commit bf5f85c

File tree

355 files changed

+14445
-10189
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

355 files changed

+14445
-10189
lines changed

‎.readthedocs.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Read the Docs configuration file for Sphinx projects
2+
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
3+
# Required
4+
version: 2
5+
# Set the OS, Python version and other tools you might need
6+
build:
7+
os: ubuntu-22.04
8+
tools:
9+
python: "3.12"
10+
# You can also specify other tool versions:
11+
# nodejs: "20"
12+
# rust: "1.70"
13+
# golang: "1.20"
14+
15+
# Build documentation in the "docs/" directory with Sphinx
16+
sphinx:
17+
configuration: sphinx/source/conf.py
18+
19+
# You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
20+
# builder: "dirhtml"
21+
# Fail on all warnings to avoid broken references
22+
# fail_on_warning: true
23+
24+
25+
# Optionally build your docs in additional formats such as PDF and ePub
26+
# formats:
27+
# - pdf
28+
# - epub
29+
30+
# Optional but recommended, declare the Python requirements required
31+
# to build your documentation
32+
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
33+
34+
python:
35+
install:
36+
- requirements: sphinx/requirements.txt

‎Makefile.am‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
SUBDIRS = include src app test sphinx 3rdparty scripts sim
1+
SUBDIRS = include src app test sphinx 3rdparty scripts
2+
#sim
23
nobase_include_HEADERS = chimbuko_config.h
34

45
install-provdb-python:

‎app/bpfile_replay.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
#include <regex>
99
#include <algorithm>
1010
#include <climits>
11-
#include <chimbuko/util/string.hpp>
12-
#include <chimbuko/util/ADIOS2parseUtils.hpp>
13-
#include <chimbuko/verbose.hpp>
11+
#include <chimbuko/core/util/string.hpp>
12+
#include <chimbuko/core/util/ADIOS2parseUtils.hpp>
13+
#include <chimbuko/core/verbose.hpp>
1414

1515
using namespace chimbuko;
1616

‎app/driver.cpp‎

Lines changed: 35 additions & 276 deletions
Large diffs are not rendered by default.

‎app/driver_multirank.cpp‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
1+
#if 0 //DISABLED DURING REFACTOR
2+
3+
14
#include<chimbuko_config.h>
25
#ifdef USE_MPI
36
#include<mpi.h>
47
#endif
5-
#include "chimbuko/chimbuko.hpp"
6-
#include "chimbuko/verbose.hpp"
7-
#include "chimbuko/util/string.hpp"
8-
#include "chimbuko/util/commandLineParser.hpp"
9-
#include "chimbuko/util/error.hpp"
8+
#include "chimbuko/core/chimbuko.hpp"
9+
#include "chimbuko/core/verbose.hpp"
10+
#include "chimbuko/core/util/string.hpp"
11+
#include "chimbuko/core/util/commandLineParser.hpp"
12+
#include "chimbuko/core/util/error.hpp"
1013
#include <chrono>
1114
#include <cstdlib>
1215

@@ -348,3 +351,12 @@ int main(int argc, char ** argv){
348351
headProgressStream(params.base_rank) << "Multi-rank driver, base rank " << params.base_rank << " is exiting" << std::endl;
349352
return error ? 1 : 0;
350353
}
354+
355+
356+
#else
357+
358+
int main(void){
359+
return 0;
360+
}
361+
362+
#endif

‎app/hpserver.cpp‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,18 @@ int main(void){
1111

1212
#else
1313

14-
#include <chimbuko/pserver.hpp>
15-
#include <chimbuko/net/zmqme_net.hpp>
14+
#include <chimbuko/modules/performance_analysis/pserver.hpp>
15+
#include <chimbuko/core/net/zmqme_net.hpp>
1616
#ifdef USE_MPI
1717
#include <mpi.h>
1818
#endif
19-
#include <chimbuko/param/sstd_param.hpp>
20-
#include <chimbuko/util/commandLineParser.hpp>
21-
#include <chimbuko/verbose.hpp>
19+
#include <chimbuko/core/param/sstd_param.hpp>
20+
#include <chimbuko/core/util/commandLineParser.hpp>
21+
#include <chimbuko/core/verbose.hpp>
2222
#include <fstream>
2323

2424
using namespace chimbuko;
25+
using namespace chimbuko::modules::performance_analysis;
2526

2627
struct hpserverArgs{
2728
int nt;
@@ -45,19 +46,19 @@ struct hpserverArgs{
4546
hpserverArgs(): nt(-1), logdir("."), ws_addr(""), load_params_set(false), save_params_set(false), freeze_params(false), stat_send_freq(1000), stat_outputdir(""), base_port(5559)
4647
{}
4748

48-
static commandLineParser<hpserverArgs> &getParser(){
49+
static commandLineParser &getParser(hpserverArgs &instance){
4950
static bool init = false;
50-
static commandLineParser<hpserverArgs> p;
51+
static commandLineParser p;
5152
if(!init){
52-
addOptionalCommandLineArg(p, nt, "Set the number of RPC handler threads (max-2 by default)");
53-
addOptionalCommandLineArg(p, logdir, "Set the output log directory (default: job directory)");
54-
addOptionalCommandLineArg(p, ws_addr, "Provide the address of the visualization module (aka webserver). If not provided no information will be sent to the visualization");
55-
addOptionalCommandLineArg(p, stat_outputdir, "Optionally provide a directory where the stat data will be written alongside/in place of sending to the viz module (default: unused");
56-
addOptionalCommandLineArgWithFlag(p, load_params, load_params_set, "Load previously computed anomaly algorithm parameters from file");
57-
addOptionalCommandLineArgWithFlag(p, save_params, save_params_set, "Save anomaly algorithm parameters to file");
58-
addOptionalCommandLineArg(p, freeze_params, "Fix the anomaly algorithm parameters, preventing updates from the AD. Use in conjunction with -load_params. Value should be 'true' or 'false' (or 0/1)");
59-
addOptionalCommandLineArg(p, stat_send_freq, "The frequency in ms at which statistics are sent to the visualization (default 1000ms)");
60-
addOptionalCommandLineArg(p, base_port, "The base port. Thread worker ports are base_port+thread_index (default 5559)");
53+
addOptionalCommandLineArg(p, instance, nt, "Set the number of RPC handler threads (max-2 by default)");
54+
addOptionalCommandLineArg(p, instance, logdir, "Set the output log directory (default: job directory)");
55+
addOptionalCommandLineArg(p, instance, ws_addr, "Provide the address of the visualization module (aka webserver). If not provided no information will be sent to the visualization");
56+
addOptionalCommandLineArg(p, instance, stat_outputdir, "Optionally provide a directory where the stat data will be written alongside/in place of sending to the viz module (default: unused");
57+
addOptionalCommandLineArgWithFlag(p, instance, load_params, load_params_set, "Load previously computed anomaly algorithm parameters from file");
58+
addOptionalCommandLineArgWithFlag(p, instance, save_params, save_params_set, "Save anomaly algorithm parameters to file");
59+
addOptionalCommandLineArg(p, instance, freeze_params, "Fix the anomaly algorithm parameters, preventing updates from the AD. Use in conjunction with -load_params. Value should be 'true' or 'false' (or 0/1)");
60+
addOptionalCommandLineArg(p, instance, stat_send_freq, "The frequency in ms at which statistics are sent to the visualization (default 1000ms)");
61+
addOptionalCommandLineArg(p, instance, base_port, "The base port. Thread worker ports are base_port+thread_index (default 5559)");
6162
init = true;
6263
}
6364
return p;
@@ -75,11 +76,11 @@ int main (int argc, char ** argv){
7576

7677

7778
hpserverArgs args;
78-
if(argc < hpserverArgs::getParser().nMandatoryArgs()+1 || (argc == 2 && std::string(argv[1]) == "-help") ){
79-
hpserverArgs::getParser().help(std::cout);
80-
return argc < hpserverArgs::getParser().nMandatoryArgs()+1 ? 1 : 0;
79+
if(argc < hpserverArgs::getParser(args).nMandatoryArgs()+1 || (argc == 2 && std::string(argv[1]) == "-help") ){
80+
hpserverArgs::getParser(args).help(std::cout);
81+
return argc < hpserverArgs::getParser(args).nMandatoryArgs()+1 ? 1 : 0;
8182
}
82-
hpserverArgs::getParser().parse(args, argc-1, (const char**)(argv+1));
83+
hpserverArgs::getParser(args).parse(argc-1, (const char**)(argv+1));
8384

8485
if (args.nt <= 0) {
8586
args.nt = std::max(

‎app/pclient.cpp‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
//A test application that mocks part of the anomaly detection modules, acting as a client for the parameter server and sending it anomaly information
22
#include <chimbuko_config.h>
33

4-
#include "chimbuko/param/sstd_param.hpp"
5-
#include "chimbuko/message.hpp"
4+
#include "chimbuko/core/param/sstd_param.hpp"
5+
#include "chimbuko/core/message.hpp"
66

77
#ifdef _USE_MPINET
8-
#include "chimbuko/net/mpi_net.hpp"
8+
#include "chimbuko/core/net/mpi_net.hpp"
99
#else
10-
#include "chimbuko/net/zmq_net.hpp"
10+
#include "chimbuko/core/net/zmq_net.hpp"
1111
#endif
1212

1313
#ifdef USE_MPI
@@ -60,10 +60,10 @@ s */
6060
//Handshake
6161
{
6262
Message msg;
63-
msg.set_info(rank, 0, (int)MessageType::REQ_ECHO, (int)MessageKind::DEFAULT);
64-
msg.set_msg("");
63+
msg.set_info(rank, 0, (int)MessageType::REQ_ECHO, (int)BuiltinMessageKind::DEFAULT);
64+
msg.setContent("");
6565
std::string strmsg;
66-
ZMQNet::send(socket, msg.data());
66+
ZMQNet::send(socket, msg.serializeMessage());
6767
ZMQNet::recv(socket, strmsg);
6868
}
6969
#endif
@@ -92,30 +92,27 @@ s */
9292
}
9393
// set message
9494
msg.clear();
95-
msg.set_info(rank, 0, MessageType::REQ_ADD, MessageKind::PARAMETERS, iFrame);
96-
msg.set_msg(l_param.serialize(), false);
95+
msg.set_info(rank, 0, MessageType::REQ_ADD, BuiltinMessageKind::PARAMETERS, iFrame);
96+
msg.setContent(l_param.serialize());
9797

9898
#ifdef _USE_MPINET
9999
// send to server
100-
MPINet::send(server, msg.data(), 0, MessageType::REQ_ADD, msg.count());
100+
MPINet::send(server, msg.serializeMessage(), 0, MessageType::REQ_ADD, msg.count());
101101

102102
// receive reply
103103
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, server, &status);
104104
//MPI_Probe(0, MessageType::REP_ADD, server, &status);
105105
std::cout << "receive reply from server\n";
106106
MPI_Get_count(&status, MPI_BYTE, &count);
107107
msg.clear();
108-
msg.set_msg(
109-
MPINet::recv(server, status.MPI_SOURCE, status.MPI_TAG, count),
110-
true
111-
);
108+
msg.deserializeMessage(MPINet::recv(server, status.MPI_SOURCE, status.MPI_TAG, count));
112109
#else
113-
ZMQNet::send(socket, msg.data());
110+
ZMQNet::send(socket, msg.serializeMessage());
114111

115112
msg.clear();
116113
std::string strmsg;
117114
ZMQNet::recv(socket, strmsg);
118-
msg.set_msg(strmsg, true);
115+
msg.deserializeMessage(strmsg);
119116
#endif
120117
//g_param.assign(msg.data_buffer());
121118
}
@@ -127,8 +124,8 @@ s */
127124
#ifdef _USE_MPINET
128125
if (rank == 0) {
129126
msg.clear();
130-
msg.set_info(rank, 0, (int)MessageType::REQ_GET, MessageKind::SSTD);
131-
MPINet::send(server, msg.data(), 0, MessageType::REQ_GET, msg.count());
127+
msg.set_info(rank, 0, (int)MessageType::REQ_GET, BuiltinMessageKind::SSTD);
128+
MPINet::send(server, msg.serializeMessage(), 0, MessageType::REQ_GET, msg.count());
132129

133130
// receive reply
134131
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, server, &status);
@@ -142,17 +139,17 @@ s */
142139
c_param.update(msg.data_buffer(), false);
143140

144141
msg.clear();
145-
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)MessageKind::DEFAULT);
142+
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)BuiltinMessageKind::DEFAULT);
146143
msg.set_msg(MessageCmd::QUIT);
147144
MPINet::send(server, msg.data(), 0, MessageType::REQ_QUIT, msg.count());
148145
}
149146
#else
150147
msg.clear();
151-
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)MessageKind::DEFAULT);
152-
msg.set_msg("");
148+
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)BuiltinMessageKind::DEFAULT);
149+
msg.setContent("");
153150
std::string strmsg;
154151
std::cout << "pclient rank " << rank << " sending disconnect notification" << std::endl;
155-
ZMQNet::send(socket, msg.data());
152+
ZMQNet::send(socket, msg.serializeMessage());
156153
std::cout << "pclient rank " << rank << " waiting for disconnect notification response" << std::endl;
157154
ZMQNet::recv(socket, strmsg);
158155
std::cout << "pclient rank " << rank << " exiting" << std::endl;

‎app/pclient_stats.cpp‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
//A test application that mocks part of the anomaly detection modules, acting as a client for the parameter server and sending it function statistics information
22
#include <chimbuko_config.h>
33

4-
#include "chimbuko/param/sstd_param.hpp"
5-
#include "chimbuko/message.hpp"
6-
#include "chimbuko/ad/AnomalyData.hpp"
7-
#include "chimbuko/ad/ADLocalFuncStatistics.hpp"
4+
#include "chimbuko/core/param/sstd_param.hpp"
5+
#include "chimbuko/core/message.hpp"
6+
#include "chimbuko/modules/performance_analysis/ad/AnomalyData.hpp"
7+
#include "chimbuko/modules/performance_analysis/ad/ADLocalFuncStatistics.hpp"
8+
#include "chimbuko/modules/performance_analysis/pserver/PScommon.hpp"
89

910
#ifdef _USE_MPINET
10-
#include "chimbuko/net/mpi_net.hpp"
11+
#include "chimbuko/core/net/mpi_net.hpp"
1112
#else
12-
#include "chimbuko/net/zmq_net.hpp"
13+
#include "chimbuko/core/net/zmq_net.hpp"
1314
#endif
1415

1516
#ifdef USE_MPI
@@ -21,6 +22,7 @@
2122
#include <random>
2223

2324
using namespace chimbuko;
25+
using namespace chimbuko::modules::performance_analysis;
2426

2527
int main (int argc, char** argv)
2628
{
@@ -52,10 +54,10 @@ int main (int argc, char** argv)
5254
//Handshake
5355
{
5456
Message msg;
55-
msg.set_info(rank, 0, (int)MessageType::REQ_ECHO, (int)MessageKind::DEFAULT);
56-
msg.set_msg("");
57+
msg.set_info(rank, 0, (int)MessageType::REQ_ECHO, (int)BuiltinMessageKind::DEFAULT);
58+
msg.setContent("");
5759
std::string strmsg;
58-
ZMQNet::send(socket, msg.data());
60+
ZMQNet::send(socket, msg.serializeMessage());
5961
ZMQNet::recv(socket, strmsg);
6062
}
6163
#endif
@@ -86,22 +88,20 @@ int main (int argc, char** argv)
8688
// create message
8789
msg.clear();
8890
msg.set_info(rank, 0, MessageType::REQ_ADD, MessageKind::ANOMALY_STATS, step);
89-
msg.set_msg(
90-
fstat.net_serialize(), false
91-
);
91+
msg.setContent(fstat.net_serialize());
9292

9393
#ifdef _USE_MPINET
9494
throw std::runtime_error("Not implemented yet.");
9595
#else
9696
// send message to parameter server
97-
ZMQNet::send(socket, msg.data());
97+
ZMQNet::send(socket, msg.serializeMessage());
9898
// std::cout << "Rank: " << rank << " sent " << step << "-th message!" << std::endl;
9999

100100
// receive reply
101101
msg.clear();
102102
strmsg.clear();
103103
ZMQNet::recv(socket, strmsg);
104-
msg.set_msg(strmsg, true);
104+
msg.deserializeMessage(strmsg);
105105
// std::cout << "Rank: " << rank << " receive " << step << "-th reply" << std::endl;
106106
#endif
107107

@@ -117,10 +117,10 @@ int main (int argc, char** argv)
117117
throw std::runtime_error("Not implemented yet.");
118118
#else
119119
msg.clear();
120-
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)MessageKind::DEFAULT);
121-
msg.set_msg("");
120+
msg.set_info(rank, 0, (int)MessageType::REQ_QUIT, (int)BuiltinMessageKind::DEFAULT);
121+
msg.setContent("");
122122
std::cout << "pclient_stats rank " << rank << " sending disconnect notification" << std::endl;
123-
ZMQNet::send(socket, msg.data());
123+
ZMQNet::send(socket, msg.serializeMessage());
124124
std::cout << "pclient_stats rank " << rank << " waiting for disconnect notification response" << std::endl;
125125
ZMQNet::recv(socket, strmsg);
126126
std::cout << "pclient_stats rank " << rank << " exiting" << std::endl;

0 commit comments

Comments
 (0)