std::shared_ptr<middle_t> mid,

std::vector<std::shared_ptr<output_t>> outs,

options_t const &options)

: m_dependency_manager(std::move(dependency_manager)), m_mid(std::move(mid)),

m_outs(std::move(outs)), m_conninfo(options.database_options.conninfo()),

m_num_procs(options.num_procs), m_append(options.append),

m_droptemp(options.droptemp), m_parallel_indexing(options.parallel_indexing),

m_with_extra_attrs(options.extra_attributes)

{

assert(m_dependency_manager);

assert(m_mid);

assert(!m_outs.empty());

{

m_mid->node_set(node);

if (m_with_extra_attrs || !node.tags().empty()) {

for (auto &out : m_outs) {

out->node_add(node);

}

}

{

m_mid->way_set(*way);

if (m_with_extra_attrs || !way->tags().empty()) {

for (auto &out : m_outs) {

out->way_add(way);

}

}

{

m_mid->relation_set(rel);

if (m_with_extra_attrs || !rel.tags().empty()) {

for (auto &out : m_outs) {

out->relation_add(rel);

}

}

{

auto &slim = slim_middle();

slim.node_delete(node.id());

slim.node_set(node);

for (auto &out : m_outs) {

out->node_modify(node);

}

m_dependency_manager->node_changed(node.id());

{

auto &slim = slim_middle();

slim.way_delete(way->id());

slim.way_set(*way);

for (auto &out : m_outs) {

out->way_modify(way);

}

m_dependency_manager->way_changed(way->id());

{

auto &slim = slim_middle();

for (auto &out : m_outs) {

out->select_relation_members(rel.id());

}

slim.relation_delete(rel.id());

slim.relation_set(rel);

for (auto &out : m_outs) {

out->relation_modify(rel);

}

{

for (auto &out : m_outs) {

out->node_delete(id);

}

slim_middle().node_delete(id);

{

for (auto &out : m_outs) {

out->way_delete(id);

}

slim_middle().way_delete(id);

{

for (auto &out : m_outs) {

out->relation_delete(id);

}

slim_middle().relation_delete(id);

{

for (auto &out : m_outs) {

out->start();

}

{

using output_vec_t = std::vector<std::shared_ptr<output_t>>;

multithreaded_processor(std::string const &conninfo,

std::shared_ptr<middle_t> const &mid,

output_vec_t outs, size_t thread_count)

: m_outputs(std::move(outs))

{

assert(!m_outputs.empty());

// For each thread we create clones of all the outputs.

m_clones.resize(thread_count);

for (size_t i = 0; i < thread_count; ++i) {

auto const midq = mid->get_query_instance();

auto copy_thread = std::make_shared<db_copy_thread_t>(conninfo);

for (auto const &out : m_outputs) {

if (out->need_forward_dependencies()) {

m_clones[i].push_back(out->clone(midq, copy_thread));

} else {

m_clones[i].emplace_back(nullptr);

}

}

}

}

/**

void process_ways(idlist_t &&list)

{

process_queue("way", std::move(list), &output_t::pending_way);

}

/**

void process_relations(idlist_t &&list)

{

process_queue("relation", std::move(list), &output_t::pending_relation);

}

/**

void process_relations_stage1c(idlist_t &&list)

{

process_queue("relation", std::move(list),

&output_t::pending_relation_stage1c);

}

/**

void merge_expire_trees()

{

std::size_t n = 0;

for (auto const &output : m_outputs) {

for (auto const &clone : m_clones) {

assert(n < clone.size());

if (clone[n]) {

output->merge_expire_trees(clone[n].get());

}

}

++n;

}

}

/// Get the next id from the queue.

static osmid_t pop_id(idlist_t *queue, std::mutex *mutex)

{

osmid_t id = 0;

std::lock_guard<std::mutex> const lock{*mutex};

if (!queue->empty()) {

id = queue->back();

queue->pop_back();

}

return id;

}

// Pointer to a member function of output_t taking an osm_id

using output_member_fn_ptr = void (output_t::*)(osmid_t);

/**

static void run(output_vec_t const &outputs, idlist_t *queue,

std::mutex *mutex, output_member_fn_ptr func)

{

while (osmid_t const id = pop_id(queue, mutex)) {

for (auto const &output : outputs) {

if (output) {

(output.get()->*func)(id);

}

}

}

for (auto const &output : outputs) {

if (output) {

output->sync();

}

}

}

/// Runs in a worker thread: Update progress display once per second.

static void print_stats(idlist_t *queue, std::mutex *mutex)

{

size_t queue_size;

do {

mutex->lock();

queue_size = queue->size();

mutex->unlock();

fmt::print(stderr, "\rLeft to process: {}...", queue_size);

std::this_thread::sleep_for(std::chrono::seconds{1});

} while (queue_size > 0);

}

void process_queue(char const *type, idlist_t list,

output_member_fn_ptr function)

{

auto const ids_queued = list.size();

fmt::print(stderr, "\nGoing over pending {}s...\n", type);

fmt::print(stderr, "\t{} {}s are pending\n", ids_queued, type);

fmt::print(stderr, "\nUsing {} helper-processes\n", m_clones.size());

util::timer_t timer;

std::vector<std::future<void>> workers;

for (auto const &clone : m_clones) {

workers.push_back(std::async(std::launch::async, run,

std::cref(clone), &list, &m_mutex,

function));

}

workers.push_back(

std::async(std::launch::async, print_stats, &list, &m_mutex));

for (auto &worker : workers) {

try {

worker.get();

} catch (...) {

// Drain the queue, so that the other workers finish early.

m_mutex.lock();

list.clear();

m_mutex.unlock();

throw;

}

}

timer.stop();

fmt::print(stderr, "\rFinished processing {} {}s in {} s\n\n",

ids_queued, type, timer.elapsed());

if (timer.elapsed() > 0) {

fmt::print(stderr,

"{} pending {}s took {}s at a rate of {:.2f}/s\n",

ids_queued, type, timer.elapsed(),

timer.per_second(ids_queued));

}

}

/// Clones of all outputs, one vector of clones per thread.

std::vector<output_vec_t> m_clones;

/// All outputs.

output_vec_t m_outputs;

/// Mutex to make sure worker threads coordinate access to queue.

std::mutex m_mutex;

osmium::Box const &bbox) const

{

if (!m_append && file.has_multiple_object_versions()) {

throw std::runtime_error{

"Reading an OSM change file only works in append mode."};

}

fmt::print(stderr, "Using {} parser.\n",

osmium::io::as_string(file.format()));

check_order_t check_order{};

input_handler_t handler{bbox, m_append, this};

osmium::io::Reader reader{file};

osmium::apply(reader, check_order, handler);

reader.close();

return handler.progress();

{

multithreaded_processor proc{m_conninfo, m_mid, m_outs,

(std::size_t)m_num_procs};

// stage 1b processing: process parents of changed objects

if (m_dependency_manager->has_pending()) {

proc.process_ways(m_dependency_manager->get_pending_way_ids());

proc.process_relations(

m_dependency_manager->get_pending_relation_ids());

proc.merge_expire_trees();

}

// stage 1c processing: mark parent relations of marked objects as changed

for (auto &out : m_outs) {

for (auto const id : out->get_marked_way_ids()) {

m_dependency_manager->way_changed(id);

}

}

// process parent relations of marked ways

if (m_dependency_manager->has_pending()) {

proc.process_relations_stage1c(

m_dependency_manager->get_pending_relation_ids());

}

{

for (auto &out : m_outs) {

out->reprocess_marked();

}

{

// All the intensive parts of this are long-running PostgreSQL commands.

// They will be run in a thread pool.

thread_pool_t pool{m_parallel_indexing ? m_num_procs : 1};

if (m_droptemp) {

// When dropping middle tables, make sure they are gone before

// indexing starts.

m_mid->stop(pool);

}

for (auto &out : m_outs) {

out->stop(&pool);

}

if (!m_droptemp) {

// When keeping middle tables, there is quite a large index created

// which is better done after the output tables have been copied.

// Note that --disable-parallel-indexing needs to be used to really

// force the order.

m_mid->stop(pool);

}

// Waiting here for pool to execute all tasks. If one of them throws an

// exception, this will throw.

pool.check_for_exceptions();

{

/* Commit the transactions, so that multiple processes can

m_mid->commit();

for (auto &out : m_outs) {

out->sync();

}

if (m_append) {

process_dependents();

}

reprocess_marked();

postprocess_database();