Skip to content

Commit 5a4d097

Browse files
e-n-fe-n-f
authored
Merge pull request #43 from mapbox/tippecanoe-fixes
Pick up tilestats fixes and other changes from Tippecanoe
2 parents 259ec79 + bf317d2 commit 5a4d097

File tree

5 files changed

+155
-72
lines changed

5 files changed

+155
-72
lines changed

‎tile.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1531,7 +1531,7 @@ int main(int argc, char **argv) {
15311531

15321532
std::map<std::string, layermap_entry> lm = merge_layermaps(layermaps);
15331533

1534-
mbtiles_write_metadata(outdb, NULL, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap, outfile);
1534+
mbtiles_write_metadata(outdb, NULL, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap, outfile, true);
15351535

15361536
write_meta(zoom_max, outdb);
15371537

‎tippecanoe/mbtiles.cpp

Lines changed: 81 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -85,26 +85,21 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data,
8585
}
8686
}
8787

88-
static void quote(std::string *buf, const char *s) {
89-
char tmp[strlen(s) * 8 + 1];
90-
char *out = tmp;
91-
92-
for (; *s != '\0'; s++) {
93-
unsigned char ch = (unsigned char) *s;
88+
static void quote(std::string &buf, std::string const &s) {
89+
for (size_t i = 0; i < s.size(); i++) {
90+
unsigned char ch = s[i];
9491

9592
if (ch == '\\' || ch == '\"') {
96-
*out++ = '\\';
97-
*out++ = ch;
93+
buf.push_back('\\');
94+
buf.push_back(ch);
9895
} else if (ch < ' ') {
99-
sprintf(out, "\\u%04x", ch);
100-
out = out + strlen(out);
96+
char tmp[7];
97+
sprintf(tmp, "\\u%04x", ch);
98+
buf.append(std::string(tmp));
10199
} else {
102-
*out++ = ch;
100+
buf.push_back(ch);
103101
}
104102
}
105-
106-
*out = '\0';
107-
buf->append(tmp, strlen(tmp));
108103
}
109104

110105
void aprintf(std::string *buf, const char *format, ...) {
@@ -142,7 +137,7 @@ bool type_and_string::operator!=(const type_and_string &o) const {
142137
return false;
143138
}
144139

145-
std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
140+
std::string tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t elements) {
146141
// Consolidate layers/attributes whose names are truncated
147142
std::vector<std::map<std::string, layermap_entry>> lmv;
148143
lmv.push_back(layermap1);
@@ -166,7 +161,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
166161
out.append("\t\t{\n");
167162

168163
out.append("\t\t\t\"layer\": \"");
169-
quote(&out, layer.first.c_str());
164+
quote(out, layer.first.c_str());
170165
out.append("\",\n");
171166

172167
out.append("\t\t\t\"count\": ");
@@ -181,7 +176,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
181176
}
182177

183178
out.append("\t\t\t\"geometry\": \"");
184-
quote(&out, geomtype.c_str());
179+
quote(out, geomtype.c_str());
185180
out.append("\",\n");
186181

187182
size_t attrib_count = layer.second.file_keys.size();
@@ -197,7 +192,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
197192

198193
size_t attrs = 0;
199194
for (auto attribute : layer.second.file_keys) {
200-
if (attrs == 100) {
195+
if (attrs == elements) {
201196
break;
202197
}
203198
if (attrs != 0) {
@@ -208,7 +203,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
208203
out.append("\t\t\t\t{\n");
209204

210205
out.append("\t\t\t\t\t\"attribute\": \"");
211-
quote(&out, attribute.first.c_str());
206+
quote(out, attribute.first.c_str());
212207
out.append("\",\n");
213208

214209
size_t val_count = attribute.second.sample_values.size();
@@ -238,30 +233,36 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
238233
}
239234

240235
out.append("\t\t\t\t\t\"type\": \"");
241-
quote(&out, type_str.c_str());
236+
quote(out, type_str.c_str());
242237
out.append("\",\n");
243238

244239
out.append("\t\t\t\t\t\"values\": [\n");
245240

246241
size_t vals = 0;
247242
for (auto value : attribute.second.sample_values) {
248-
if (vals == 100) {
243+
if (vals == elements) {
249244
break;
250245
}
251-
if (vals != 0) {
252-
out.append(",\n");
253-
}
254-
vals++;
255246

256247
if (value.type == mvt_double || value.type == mvt_bool) {
248+
if (vals != 0) {
249+
out.append(",\n");
250+
}
251+
vals++;
252+
257253
out.append("\t\t\t\t\t\t");
258254
out.append(value.string);
259255
} else {
260256
std::string trunc = truncate16(value.string, 256);
261257

262258
if (trunc.size() == value.string.size()) {
259+
if (vals != 0) {
260+
out.append(",\n");
261+
}
262+
vals++;
263+
263264
out.append("\t\t\t\t\t\t\"");
264-
quote(&out, value.string.c_str());
265+
quote(out, value.string.c_str());
265266
out.append("\"");
266267
}
267268
}
@@ -304,7 +305,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
304305
return out2;
305306
}
306307

307-
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description) {
308+
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats) {
308309
char *sql, *err;
309310

310311
sqlite3 *db = outdb;
@@ -412,57 +413,66 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
412413
sqlite3_free(sql);
413414

414415
if (vector) {
415-
std::string buf("{");
416-
aprintf(&buf, "\"vector_layers\": [ ");
416+
size_t elements = 100;
417+
std::string buf;
417418

418-
std::vector<std::string> lnames;
419-
for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
420-
lnames.push_back(ai->first);
421-
}
419+
{
420+
buf = "{";
421+
aprintf(&buf, "\"vector_layers\": [ ");
422422

423-
for (size_t i = 0; i < lnames.size(); i++) {
424-
if (i != 0) {
425-
aprintf(&buf, ", ");
423+
std::vector<std::string> lnames;
424+
for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
425+
lnames.push_back(ai->first);
426426
}
427427

428-
auto fk = layermap.find(lnames[i]);
429-
aprintf(&buf, "{ \"id\": \"");
430-
quote(&buf, lnames[i].c_str());
431-
aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);
432-
433-
bool first = true;
434-
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
435-
if (first) {
436-
first = false;
437-
} else {
428+
for (size_t i = 0; i < lnames.size(); i++) {
429+
if (i != 0) {
438430
aprintf(&buf, ", ");
439431
}
440432

441-
aprintf(&buf, "\"");
442-
quote(&buf, j->first.c_str());
433+
auto fk = layermap.find(lnames[i]);
434+
aprintf(&buf, "{ \"id\": \"");
435+
quote(buf, lnames[i]);
436+
aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);
437+
438+
bool first = true;
439+
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
440+
if (first) {
441+
first = false;
442+
} else {
443+
aprintf(&buf, ", ");
444+
}
445+
446+
aprintf(&buf, "\"");
447+
quote(buf, j->first.c_str());
443448

444-
int type = 0;
445-
for (auto s : j->second.sample_values) {
446-
type |= (1 << s.type);
447-
}
449+
int type = 0;
450+
for (auto s : j->second.sample_values) {
451+
type |= (1 << s.type);
452+
}
448453

449-
if (type == (1 << mvt_double)) {
450-
aprintf(&buf, "\": \"Number\"");
451-
} else if (type == (1 << mvt_bool)) {
452-
aprintf(&buf, "\": \"Boolean\"");
453-
} else if (type == (1 << mvt_string)) {
454-
aprintf(&buf, "\": \"String\"");
455-
} else {
456-
aprintf(&buf, "\": \"Mixed\"");
454+
if (type == (1 << mvt_double)) {
455+
aprintf(&buf, "\": \"Number\"");
456+
} else if (type == (1 << mvt_bool)) {
457+
aprintf(&buf, "\": \"Boolean\"");
458+
} else if (type == (1 << mvt_string)) {
459+
aprintf(&buf, "\": \"String\"");
460+
} else {
461+
aprintf(&buf, "\": \"Mixed\"");
462+
}
457463
}
464+
465+
aprintf(&buf, "} }");
458466
}
459467

460-
aprintf(&buf, "} }");
461-
}
468+
aprintf(&buf, " ]");
469+
470+
if (do_tilestats && elements > 0) {
471+
aprintf(&buf, ",\"tilestats\": %s", tilestats(layermap, elements).c_str());
472+
}
462473

463-
aprintf(&buf, " ],");
464-
aprintf(&buf, "\"tilestats\": %s", tilestats(layermap).c_str());
465-
aprintf(&buf, "}");
474+
aprintf(&buf, "}");
475+
}
466476

467477
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str());
468478
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
@@ -490,8 +500,8 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
490500
while (sqlite3_step(stmt) == SQLITE_ROW) {
491501
std::string key, value;
492502

493-
quote(&key, (const char *) sqlite3_column_text(stmt, 0));
494-
quote(&value, (const char *) sqlite3_column_text(stmt, 1));
503+
quote(key, (const char *) sqlite3_column_text(stmt, 0));
504+
quote(value, (const char *) sqlite3_column_text(stmt, 1));
495505

496506
if (!first) {
497507
fprintf(fp, ",\n");
@@ -536,6 +546,10 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
536546

537547
for (size_t i = 0; i < maps.size(); i++) {
538548
for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
549+
if (map->second.points + map->second.lines + map->second.polygons == 0) {
550+
continue;
551+
}
552+
539553
std::string layername = map->first;
540554
if (trunc) {
541555
layername = truncate16(layername, 256);

‎tippecanoe/mbtiles.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
3939

4040
void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
4141

42-
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description);
42+
void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats);
4343

4444
void mbtiles_close(sqlite3 *outdb, const char *pgm);
4545

‎tippecanoe/mvt.cpp

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#include <vector>
55
#include <map>
66
#include <zlib.h>
7+
#include <errno.h>
8+
#include <limits.h>
79
#include "mvt.hpp"
810
#include "protozero/varint.hpp"
911
#include "protozero/pbf_reader.hpp"
@@ -416,3 +418,69 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
416418
feature.tags.push_back(ko);
417419
feature.tags.push_back(vo);
418420
}
421+
422+
static int is_integer(const char *s, long long *v) {
423+
errno = 0;
424+
char *endptr;
425+
426+
*v = strtoll(s, &endptr, 0);
427+
if (*v == 0 && errno != 0) {
428+
return 0;
429+
}
430+
if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE)) {
431+
return 0;
432+
}
433+
if (*endptr != '\0') {
434+
// Special case: If it is an integer followed by .0000 or similar,
435+
// it is still an integer
436+
437+
if (*endptr != '.') {
438+
return 0;
439+
}
440+
endptr++;
441+
for (; *endptr != '\0'; endptr++) {
442+
if (*endptr != '0') {
443+
return 0;
444+
}
445+
}
446+
447+
return 1;
448+
}
449+
450+
return 1;
451+
}
452+
453+
mvt_value stringified_to_mvt_value(int type, const char *s) {
454+
mvt_value tv;
455+
456+
if (type == mvt_double) {
457+
long long v;
458+
if (is_integer(s, &v)) {
459+
if (v >= 0) {
460+
tv.type = mvt_int;
461+
tv.numeric_value.int_value = v;
462+
} else {
463+
tv.type = mvt_sint;
464+
tv.numeric_value.sint_value = v;
465+
}
466+
} else {
467+
double d = atof(s);
468+
469+
if (d == (float) d) {
470+
tv.type = mvt_float;
471+
tv.numeric_value.float_value = d;
472+
} else {
473+
tv.type = mvt_double;
474+
tv.numeric_value.double_value = d;
475+
}
476+
}
477+
} else if (type == mvt_bool) {
478+
tv.type = mvt_bool;
479+
tv.numeric_value.bool_value = (s[0] == 't');
480+
} else {
481+
tv.type = mvt_string;
482+
tv.string_value = s;
483+
}
484+
485+
return tv;
486+
}

‎tippecanoe/mvt.hpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ enum mvt_operation {
1717
};
1818

1919
struct mvt_geometry {
20-
int x;
21-
int y;
20+
long long x;
21+
long long y;
2222
int /* mvt_operation */ op;
2323

2424
mvt_geometry(int op, long long x, long long y);
@@ -86,7 +86,7 @@ struct mvt_layer {
8686
std::vector<mvt_feature> features;
8787
std::vector<std::string> keys;
8888
std::vector<mvt_value> values;
89-
int extent;
89+
long long extent;
9090

9191
// Add a key-value pair to a feature, using this layer's constant pool
9292
void tag(mvt_feature &feature, std::string key, mvt_value value);
@@ -108,4 +108,5 @@ int decompress(std::string const &input, std::string &output);
108108
int compress(std::string const &input, std::string &output);
109109
int dezig(unsigned n);
110110

111+
mvt_value stringified_to_mvt_value(int type, const char *s);
111112
#endif

0 commit comments

Comments
 (0)