Skip to content

Commit 0c5add6

Browse files
mmd-osmmmd-osm
committed
Brotli compression prototype
1 parent ea3e595 commit 0c5add6

12 files changed

+349
-4
lines changed

‎CMakeLists.txt

+7
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ endif()
3737
###############
3838
option(ENABLE_APIDB "Enable APIDB backend, as used by the OSM servers" ON)
3939
option(ENABLE_YAJL "Enable JSON output with the YAJL library" ON)
40+
option(ENABLE_BROTLI "Enable Brotli library" ON)
4041
option(ENABLE_FMT_HEADER "Enable FMT header only mode" ON)
4142
option(ENABLE_COVERAGE "Compile with coverage info collection" OFF)
4243
option(ENABLE_PROFILING "Compile with profiling" OFF)
@@ -102,6 +103,12 @@ endif()
102103
target_compile_definitions(cgimap_common_compiler_options INTERFACE
103104
HAVE_YAJL=$<BOOL:${YAJL_FOUND}>)
104105

106+
if(ENABLE_BROTLI)
107+
find_package(Brotli COMPONENTS encoder decoder common REQUIRED)
108+
endif()
109+
target_compile_definitions(cgimap_common_compiler_options INTERFACE
110+
HAVE_BROTLI=$<BOOL:${Brotli_FOUND}>)
111+
105112
find_package(Fcgi REQUIRED)
106113
target_compile_definitions(cgimap_common_compiler_options INTERFACE
107114
HAVE_FCGI=$<BOOL:${Fcgi_FOUND}>)

‎cmake/FindBrotli.cmake

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
# A simple FindBrotli package for Cmake's find_package function.
2+
# Note: This find package doesn't have version support, as the version file doesn't seem to be installed on most systems.
3+
#
4+
# If you want to find the static packages instead of shared (the default), define BROTLI_USE_STATIC_LIBS as TRUE.
5+
# The targets will have the same names, but it will use the static libs.
6+
#
7+
# Valid find_package COMPONENTS names: "decoder", "encoder", and "common"
8+
# Note that if you're requiring "decoder" or "encoder", then "common" will be automatically added as required.
9+
#
10+
# Defines the libraries (if found): Brotli::decoder, Brotli::encoder, Brotli::common
11+
# and the includes path variable: Brotli_INCLUDE_DIR
12+
#
13+
# If it's failing to find the libraries, try setting BROTLI_ROOT_DIR to the folder containing your library & include dir.
14+
15+
# If they asked for a specific version, warn/fail since we don't support it.
16+
# TODO: if they start distributing the version somewhere, implement finding it.
17+
# See https://github.com/google/brotli/issues/773#issuecomment-579133187
18+
if(Brotli_FIND_VERSION)
19+
set(_brotli_version_error_msg "FindBrotli.cmake doesn't have version support!")
20+
# If the package is required, throw a fatal error
21+
# Otherwise, if not running quietly, we throw a warning
22+
if(Brotli_FIND_REQUIRED)
23+
message(FATAL_ERROR "${_brotli_version_error_msg}")
24+
elseif(NOT Brotli_FIND_QUIETLY)
25+
message(WARNING "${_brotli_version_error_msg}")
26+
endif()
27+
endif()
28+
29+
# Since both decoder & encoder require the common lib, force its requirement..
30+
# if the user is requiring either of those other libs.
31+
if(Brotli_FIND_REQUIRED_decoder OR Brotli_FIND_REQUIRED_encoder)
32+
set(Brotli_FIND_REQUIRED_common TRUE)
33+
endif()
34+
35+
# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
36+
# Credit to FindOpenSSL.cmake for this
37+
if(BROTLI_USE_STATIC_LIBS)
38+
set(_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
39+
if(WIN32)
40+
set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
41+
else()
42+
set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
43+
endif()
44+
endif()
45+
46+
# Make PkgConfig optional, since some users (mainly Windows) don't have it.
47+
# But it's a lot more clean than manually using find_library.
48+
find_package(PkgConfig QUIET)
49+
50+
# Only used if the PkgConfig libraries aren't used.
51+
find_path(Brotli_INCLUDE_DIR
52+
NAMES
53+
"brotli/decode.h"
54+
"brotli/encode.h"
55+
HINTS
56+
${BROTLI_ROOT_DIR}
57+
PATH_SUFFIXES
58+
"include"
59+
"includes"
60+
DOC "The path to Brotli's include directory."
61+
)
62+
# Hides this var from the GUI
63+
mark_as_advanced(Brotli_INCLUDE_DIR)
64+
65+
# Just used for PkgConfig stuff in the loop below
66+
set(_brotli_stat_str "")
67+
if(BROTLI_USE_STATIC_LIBS)
68+
set(_brotli_stat_str "_STATIC")
69+
endif()
70+
71+
# Each string here is "ComponentName;LiteralName" (the semi-colon is a delimiter)
72+
foreach(_listvar "common;common" "decoder;dec" "encoder;enc")
73+
# Split the component name and literal library name from the listvar
74+
list(GET _listvar 0 _component_name)
75+
list(GET _listvar 1 _libname)
76+
77+
# NOTE: We can't rely on PkgConf for static libs since the upstream static lib support is broken
78+
# See https://github.com/google/brotli/issues/795
79+
# TODO: whenever their issue is fixed upstream, remove this "AND NOT BROTLI_USE_STATIC_LIBS" check
80+
if(PKG_CONFIG_FOUND AND NOT BROTLI_USE_STATIC_LIBS)
81+
# These need to be GLOBAL for MinGW when making ALIAS libraries against them.
82+
# Have to postfix _STATIC on the name to tell PkgConfig to find the static libs.
83+
pkg_check_modules(Brotli_${_component_name}${_brotli_stat_str} QUIET GLOBAL IMPORTED_TARGET libbrotli${_libname})
84+
endif()
85+
86+
# Check if the target was already found by Pkgconf
87+
if(TARGET PkgConfig::Brotli_${_component_name}${_brotli_stat_str})
88+
# ALIAS since we don't want the PkgConfig namespace on the Cmake library (for end-users)
89+
add_library(Brotli::${_component_name} ALIAS PkgConfig::Brotli_${_component_name}${_brotli_stat_str})
90+
91+
# Tells HANDLE_COMPONENTS we found the component
92+
set(Brotli_${_component_name}_FOUND TRUE)
93+
if(Brotli_FIND_REQUIRED_${_component_name})
94+
# If the lib is required, we can add its literal path as a required var for FindPackageHandleStandardArgs
95+
# Since it won't accept the PkgConfig targets
96+
if(BROTLI_USE_STATIC_LIBS)
97+
list(APPEND _brotli_req_vars "Brotli_${_component_name}_STATIC_LIBRARIES")
98+
else()
99+
list(APPEND _brotli_req_vars "Brotli_${_component_name}_LINK_LIBRARIES")
100+
endif()
101+
endif()
102+
103+
# Skip searching for the libs with find_library since it was already found by Pkgconf
104+
continue()
105+
endif()
106+
107+
if(Brotli_FIND_REQUIRED_${_component_name})
108+
# If it's required, we can set the name used in find_library as a required var for FindPackageHandleStandardArgs
109+
list(APPEND _brotli_req_vars "Brotli_${_component_name}")
110+
endif()
111+
112+
list(APPEND _brotli_lib_names
113+
"brotli${_libname}"
114+
"libbrotli${_libname}"
115+
)
116+
if(BROTLI_USE_STATIC_LIBS)
117+
# Postfix "-static" to the libnames since we're looking for static libs
118+
list(TRANSFORM _brotli_lib_names APPEND "-static")
119+
endif()
120+
121+
find_library(Brotli_${_component_name}
122+
NAMES ${_brotli_lib_names}
123+
HINTS ${BROTLI_ROOT_DIR}
124+
PATH_SUFFIXES
125+
"lib"
126+
"lib64"
127+
"libs"
128+
"libs64"
129+
"lib/x86_64-linux-gnu"
130+
)
131+
# Hide the library variable from the Cmake GUI
132+
mark_as_advanced(Brotli_${_component_name})
133+
134+
# Unset since otherwise it'll stick around for the next loop and break things
135+
unset(_brotli_lib_names)
136+
137+
# Check if find_library found the library
138+
if(Brotli_${_component_name})
139+
# Tells HANDLE_COMPONENTS we found the component
140+
set(Brotli_${_component_name}_FOUND TRUE)
141+
142+
add_library("Brotli::${_component_name}" UNKNOWN IMPORTED)
143+
# Attach the literal library and include dir to the IMPORTED target for the end-user
144+
set_target_properties("Brotli::${_component_name}" PROPERTIES
145+
INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}"
146+
IMPORTED_LOCATION "${Brotli_${_component_name}}"
147+
)
148+
else()
149+
# Tells HANDLE_COMPONENTS we found the component
150+
set(Brotli_${_component_name}_FOUND FALSE)
151+
endif()
152+
endforeach()
153+
154+
include(FindPackageHandleStandardArgs)
155+
# Sets Brotli_FOUND, and fails the find_package(Brotli) call if it was REQUIRED but missing libs.
156+
find_package_handle_standard_args(Brotli
157+
FOUND_VAR
158+
Brotli_FOUND
159+
REQUIRED_VARS
160+
Brotli_INCLUDE_DIR
161+
${_brotli_req_vars}
162+
HANDLE_COMPONENTS
163+
)
164+
165+
# Restore the original find library ordering
166+
if(BROTLI_USE_STATIC_LIBS)
167+
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
168+
endif()

‎docker/debian/Dockerfile_bookworm

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
55

66
RUN apt-get update -qq && \
77
apt-get install -y gcc g++ make cmake \
8-
libfcgi-dev libxml2-dev libmemcached-dev \
8+
libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
99
libboost-program-options-dev libcrypto++-dev libyajl-dev \
1010
libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
1111
postgresql-15 postgresql-server-dev-all \

‎docker/debian/Dockerfile_trixie

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
55

66
RUN apt-get update -qq && \
77
apt-get install -y gcc g++ make cmake \
8-
libfcgi-dev libxml2-dev libmemcached-dev \
8+
libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
99
libboost-program-options-dev libcrypto++-dev libyajl-dev \
1010
libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
1111
postgresql-16 postgresql-server-dev-all \

‎docker/ubuntu/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
55

66
RUN apt-get update -qq && \
77
apt-get install -y gcc g++ make cmake \
8-
libfcgi-dev libxml2-dev libmemcached-dev \
8+
libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
99
libboost-program-options-dev \
1010
libcrypto++-dev libyajl-dev \
1111
libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \

‎docker/ubuntu/Dockerfile2204

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
55

66
RUN apt-get update -qq && \
77
apt-get install -y gcc g++ make cmake \
8-
libfcgi-dev libxml2-dev libmemcached-dev \
8+
libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
99
libboost-program-options-dev libcrypto++-dev libyajl-dev \
1010
libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
1111
postgresql-14 postgresql-server-dev-all \

‎include/cgimap/brotli.hpp

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* SPDX-License-Identifier: GPL-2.0-only
3+
*
4+
* This file is part of openstreetmap-cgimap (https://github.com/zerebubuth/openstreetmap-cgimap/).
5+
*
6+
* Copyright (C) 2009-2023 by the CGImap developer community.
7+
* For a full list of authors see the git log.
8+
*/
9+
10+
11+
#ifndef BROTLI_HPP
12+
#define BROTLI_HPP
13+
14+
#if HAVE_BROTLI
15+
16+
#include <functional>
17+
#include <memory>
18+
19+
#include <brotli/decode.h>
20+
#include <brotli/encode.h>
21+
22+
#include "cgimap/output_buffer.hpp"
23+
24+
25+
/**
26+
* Compresses an output stream.
27+
*/
28+
class brotli_output_buffer : public output_buffer {
29+
public:
30+
31+
brotli_output_buffer(output_buffer& o);
32+
brotli_output_buffer(const brotli_output_buffer &old) = delete;
33+
~brotli_output_buffer() override = default;
34+
int write(const char *buffer, int len) override;
35+
int written() override;
36+
int close() override;
37+
void flush() override;
38+
39+
private:
40+
int compress(const char *data, int data_length, bool last);
41+
42+
BrotliEncoderState *state_ = nullptr;
43+
std::array<uint8_t, 16384> buff;
44+
45+
output_buffer& out;
46+
// keep track of bytes written
47+
size_t bytes_in = 0;
48+
bool flushed{false};
49+
};
50+
51+
#endif
52+
53+
#endif

‎include/cgimap/http.hpp

+15
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
#ifdef HAVE_LIBZ
2222
#include "cgimap/zlib.hpp"
2323
#endif
24+
#if HAVE_BROTLI
25+
#include "cgimap/brotli.hpp"
26+
#endif
27+
2428
#include "cgimap/output_buffer.hpp"
2529

2630
/**
@@ -284,6 +288,17 @@ class gzip : public encoding {
284288
};
285289
#endif /* HAVE_LIBZ */
286290

291+
#if HAVE_BROTLI
292+
293+
class brotli : public encoding {
294+
public:
295+
brotli() : encoding("br"){}
296+
std::unique_ptr<output_buffer> buffer(output_buffer& out) override {
297+
return std::make_unique<brotli_output_buffer>(out);
298+
}
299+
};
300+
#endif
301+
287302
/*
288303
* Parses an Accept-Encoding header and returns the chosen
289304
* encoding.

‎src/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ target_sources(cgimap_core PRIVATE
1010
backend.cpp
1111
backend.cpp
1212
bbox.cpp
13+
brotli.cpp
1314
choose_formatter.cpp
1415
handler.cpp
1516
http.cpp
@@ -78,6 +79,9 @@ target_link_libraries(cgimap_core
7879
CryptoPP::CryptoPP
7980
Libmemcached::Libmemcached
8081
Argon2::Argon2
82+
$<$<BOOL:${ENABLE_BROTLI}>:Brotli::common>
83+
$<$<BOOL:${ENABLE_BROTLI}>:Brotli::encoder>
84+
$<$<BOOL:${ENABLE_BROTLI}>:Brotli::decoder>
8185
$<$<BOOL:${ENABLE_YAJL}>:YAJL::YAJL>
8286
PQXX::PQXX)
8387

0 commit comments

Comments
 (0)