Skip to content

Commit fe7ce22

Browse files
hollingerhollinger
committed
Populate pending job queue prior to cloning out the middles and outputs.
Some other bug fixes and refactor. Most tests pass now, but three still fail.
1 parent bae7808 commit fe7ce22

25 files changed

Lines changed: 381 additions & 176 deletions

‎id-tracker.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ bool id_tracker::is_marked(osmid_t id) {
158158
osmid_t id_tracker::pop_mark() {
159159
osmid_t id = impl->pop_min();
160160

161-
assert((id > impl->old_id) || (id == std::numeric_limits<osmid_t>::max()));
161+
assert((id > impl->old_id) || !id_tracker::is_valid(id));
162162
impl->old_id = id;
163163

164164
//we just go rid of one (if there were some to get rid of)
@@ -169,3 +169,5 @@ osmid_t id_tracker::pop_mark() {
169169
}
170170

171171
size_t id_tracker::size() { return impl->count; }
172+
173+
bool id_tracker::is_valid(osmid_t id) { return id != std::numeric_limits<osmid_t>::max(); }

‎id-tracker.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ struct id_tracker : public boost::noncopyable {
1515
osmid_t pop_mark();
1616
size_t size();
1717

18+
static bool is_valid(osmid_t);
19+
1820
private:
1921
struct pimpl;
2022
boost::scoped_ptr<pimpl> impl;

‎middle-pgsql.cpp‎

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -714,53 +714,36 @@ int middle_pgsql_t::ways_delete(osmid_t osm_id)
714714
return 0;
715715
}
716716

717-
void middle_pgsql_t::iterate_ways(middle_t::cb_func &callback)
717+
void middle_pgsql_t::iterate_ways(middle_t::pending_processor& pf)
718718
{
719-
// The flag we pass to indicate that the way in question might exist already in the database */
720-
int exists = Append;
721719
time_t start, end;
722720
time(&start);
721+
723722
fprintf(stderr, "\nGoing over pending ways...\n");
724723

725724
// Make sure we're out of copy mode */
726725
pgsql_endCopy( way_table );
727726

728727
if (out_options->flat_node_cache_enabled) persistent_cache.reset();
729728

730-
size_t pending_count = ways_pending_tracker->size();
731-
fprintf(stderr, "\t%zu ways are pending\n", pending_count);
732-
733-
/**
734-
* TODO
735-
* To speed up processing of pending ways, fork noProcs worker processes
736-
* each of which independently goes through an equal subset of the pending ways array
737-
*/
738-
fprintf(stderr, "\nUsing %i helper-processes\n", 1);
739-
740729
if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options,1,cache)); /* at this point we always want to be in append mode, to not delete and recreate the node cache file */
741730

742-
// some spaces at end, so that processings outputs get cleaned if already existing */
743-
fprintf(stderr, "\rHelper process %i out of %i initialised \n", 0, 1);
744-
745-
//in memory processing pending ways
731+
// enqueue the jobs
746732
osmid_t id;
747-
int count = 0;
748-
while((id = ways_pending_tracker->pop_mark()) != std::numeric_limits<osmid_t>::max())
733+
while(id_tracker::is_valid(id = ways_pending_tracker->pop_mark()))
749734
{
750-
//progress update
751-
if(count++ %1000 == 0)
752-
{
753-
time(&end);
754-
fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s", count/1000,
755-
end > start ? ((double)count / 1000.0 / (double)(end - start)) : 0);
756-
}
757-
758-
//send it to the backends, mark it done and cleanup
759-
callback(id, exists);
735+
pf.enqueue(id);
760736
}
761737

738+
size_t pending_count = pf.size();
739+
fprintf(stderr, "\t%zu ways are pending\n", pending_count);
740+
fprintf(stderr, "\nUsing %i helper-processes\n", pf.thread_count());
741+
742+
//let the threads work on them
743+
pf.process_ways();
744+
762745
time(&end);
763-
fprintf(stderr, "\rProcess %i finished processing %i ways in %i sec\n", 0, count, (int)(end - start));
746+
fprintf(stderr, "\rProcess %i finished processing %i ways in %i sec\n", 0, pending_count, (int)(end - start));
764747

765748
fprintf(stderr, "\nAll child processes exited\n");
766749

@@ -1530,3 +1513,7 @@ boost::shared_ptr<const middle_query_t> middle_pgsql_t::get_instance() const {
15301513

15311514
return boost::shared_ptr<middle_query_t>(mid);
15321515
}
1516+
1517+
size_t middle_pgsql_t::pending_count() const {
1518+
return ways_pending_tracker->size() + rels_pending_tracker->size();
1519+
}

‎middle-pgsql.hpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ struct middle_pgsql_t : public slim_middle_t {
4545
int relations_delete(osmid_t id);
4646
int relation_changed(osmid_t id);
4747

48-
void iterate_ways(cb_func &cb);
48+
void iterate_ways(middle_t::pending_processor& pf);
4949
void iterate_relations(cb_func &cb);
5050

51+
size_t pending_count() const;
52+
5153
std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
5254

5355
void *pgsql_stop_one(void *arg);

‎middle-ram.cpp‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,13 @@ void middle_ram_t::iterate_relations(middle_t::cb_func &callback)
199199
fprintf(stderr, "\rWriting relation (%u)\n", rel_out_count);
200200
}
201201

202-
void middle_ram_t::iterate_ways(middle_t::cb_func &callback)
202+
size_t middle_ram_t::pending_count() const {
203+
//TODO: keep a running count of marked pending stuff
204+
//so we dont have to iterate over the memory to know
205+
return 42;
206+
}
207+
208+
void middle_ram_t::iterate_ways(middle_t::pending_processor& pf)
203209
{
204210
int block, offset;
205211

@@ -212,21 +218,26 @@ void middle_ram_t::iterate_ways(middle_t::cb_func &callback)
212218
if (ways[block][offset].ndids) {
213219
way_out_count++;
214220
if (way_out_count % 1000 == 0)
215-
fprintf(stderr, "\rWriting way (%uk)", way_out_count/1000);
221+
fprintf(stderr, "\rEnqueuing way (%uk)", way_out_count/1000);
216222

217223
if (ways[block][offset].pending) {
218224
/* First element contains number of nodes */
219225
if (ways[block][offset].ndids[0]) {
220226
osmid_t id = block2id(block, offset);
221-
callback(id, 0);
227+
pf.enqueue(id);
222228
}
223229

224230
ways[block][offset].pending = 0;
225231
}
226232
}
227233
}
228234
}
229-
fprintf(stderr, "\rWriting way (%uk)\n", way_out_count/1000);
235+
fprintf(stderr, "\rEnqueuing way (%uk)\n", way_out_count/1000);
236+
237+
//let the threads process the ways
238+
pf.process_ways();
239+
240+
//TODO: message to show the real progress of writing the ways
230241
}
231242

232243
void middle_ram_t::release_relations()

‎middle-ram.hpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,11 @@ struct middle_ram_t : public middle_t {
4444

4545
std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
4646

47-
void iterate_ways(cb_func &cb);
47+
void iterate_ways(middle_t::pending_processor& pf);
4848
void iterate_relations(cb_func &cb);
4949

50+
size_t pending_count() const;
51+
5052
virtual boost::shared_ptr<const middle_query_t> get_instance() const;
5153
private:
5254

‎middle.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ slim_middle_t::~slim_middle_t() {
1919

2020
middle_t::cb_func::~cb_func() {
2121
}
22+
23+
middle_t::pending_processor::~pending_processor() {
24+
}

‎middle.hpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,19 @@ struct middle_t : public middle_query_t {
5252
virtual void finish(int exists) = 0;
5353
};
5454

55-
virtual void iterate_ways(cb_func &cb) = 0;
55+
struct pending_processor {
56+
virtual ~pending_processor();
57+
virtual void enqueue(osmid_t id) = 0;
58+
virtual void process_ways() = 0;
59+
virtual int thread_count() = 0;
60+
virtual int size() = 0;
61+
};
62+
63+
virtual void iterate_ways(pending_processor& pf) = 0;
5664
virtual void iterate_relations(cb_func &cb) = 0;
5765

66+
virtual size_t pending_count() const = 0;
67+
5868
const options_t* out_options;
5969
};
6070

0 commit comments

Comments
 (0)