Commit 6ea5f9b0 authored by Baptiste Daroussin's avatar Baptiste Daroussin
Browse files

Update libucl to snapshot 20160812

parent 6b85ccc6
......@@ -205,8 +205,7 @@ SET(UCLSRC src/ucl_util.c
src/ucl_hash.c
src/ucl_schema.c
src/ucl_msgpack.c
src/ucl_sexp.c
src/xxhash.c)
src/ucl_sexp.c)
SET (LIB_TYPE STATIC)
......
CC ?= gcc
DESTDIR ?= /usr/local
LD ?= gcc
C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src
C_COMMON_FLAGS ?= -fPIC -Wall -W -Wno-unused-parameter -Wno-pointer-sign -I./include -I./uthash -I./src -I./klib
MAJOR_VERSION = 0
MINOR_VERSION = 2
PATCH_VERSION = 9
......@@ -25,13 +25,12 @@ HDEPS = $(SRCDIR)/ucl_hash.h \
$(SRCDIR)/ucl_chartable.h \
$(SRCDIR)/ucl_internal.h \
$(INCLUDEDIR)/ucl.h \
$(SRCDIR)/xxhash.h
$(SRCDIR)/mum.h
OBJECTS = $(OBJDIR)/ucl_hash.o \
$(OBJDIR)/ucl_util.o \
$(OBJDIR)/ucl_parser.o \
$(OBJDIR)/ucl_emitter.o \
$(OBJDIR)/ucl_schema.o \
$(OBJDIR)/xxhash.o
$(OBJDIR)/ucl_schema.o
all: $(OBJDIR) $(OBJDIR)/$(SONAME)
......@@ -55,8 +54,6 @@ $(OBJDIR)/ucl_hash.o: $(SRCDIR)/ucl_hash.c $(HDEPS)
$(CC) -o $(OBJDIR)/ucl_hash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_hash.c
$(OBJDIR)/ucl_schema.o: $(SRCDIR)/ucl_schema.c $(HDEPS)
$(CC) -o $(OBJDIR)/ucl_schema.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/ucl_schema.c
$(OBJDIR)/xxhash.o: $(SRCDIR)/xxhash.c $(HDEPS)
$(CC) -o $(OBJDIR)/xxhash.o $(CPPFLAGS) $(COPT_FLAGS) $(CFLAGS) $(C_COMMON_FLAGS) $(SSL_CFLAGS) $(FETCH_FLAGS) -c $(SRCDIR)/xxhash.c
clean:
$(RM) $(OBJDIR)/*.o $(OBJDIR)/$(SONAME_FULL) $(OBJDIR)/$(SONAME) $(OBJDIR)/chargen $(OBJDIR)/test_basic $(OBJDIR)/test_speed $(OBJDIR)/objdump $(OBJDIR)/test_generate $(OBJDIR)/test_schema || true
......
......@@ -28,14 +28,13 @@ HDEPS = $(SRCDIR)/ucl_hash.h \
$(SRCDIR)/ucl_chartable.h \
$(SRCDIR)/ucl_internal.h \
$(INCLUDEDIR)/ucl.h \
$(SRCDIR)/xxhash.h
$(SRCDIR)/mum.h
OBJECTS = $(OBJDIR)/ucl_hash.o \
$(OBJDIR)/ucl_util.o \
$(OBJDIR)/ucl_parser.o \
$(OBJDIR)/ucl_emitter.o \
$(OBJDIR)/ucl_emitter_utils.o \
$(OBJDIR)/ucl_schema.o \
$(OBJDIR)/xxhash.o
$(OBJDIR)/ucl_schema.o
all: $(OBJDIR) $(OBJDIR)/$(SONAME)
......
......@@ -12,7 +12,7 @@
- [Named keys hierarchy](#named-keys-hierarchy)
- [Convenient numbers and booleans](#convenient-numbers-and-booleans)
- [General improvements](#general-improvements)
- [Commments](#commments)
- [Comments](#comments)
- [Macros support](#macros-support)
- [Variables support](#variables-support)
- [Multiline strings](#multiline-strings)
......@@ -21,7 +21,7 @@
- [Performance](#performance)
- [Conclusion](#conclusion)
## Introduction
## Introduction
This document describes the main features and principles of the configuration
language called `UCL` - universal configuration language.
......@@ -47,7 +47,7 @@ section {
string = "something";
subsection {
host = {
host = "hostname";
host = "hostname";
port = 900;
}
host = {
......@@ -163,9 +163,9 @@ section {
}
}
```
Plain definitions may be more complex and contain more than a single level of nested objects:
```nginx
section "blah" "foo" {
key = value;
......@@ -174,7 +174,7 @@ section "blah" "foo" {
is presented as:
```nginx
```nginx
section {
blah {
foo {
......@@ -196,17 +196,17 @@ section {
## General improvements
### Commments
### Comments
UCL supports different style of comments:
* single line: `#`
* single line: `#`
* multiline: `/* ... */`
Multiline comments may be nested:
```c
# Sample single line comment
/*
/*
some comment
/* nested comment */
end of comment
......@@ -263,7 +263,7 @@ all files that matches the specified pattern (normally the format of patterns is
for your operating system). This option is meaningless for URL includes.
* `url` (default: **true**) - allow URL includes.
* `path` (default: empty) - A UCL_ARRAY of directories to search for the include file.
Search ends after the first patch, unless `glob` is true, then all matches are included.
Search ends after the first match, unless `glob` is true, then all matches are included.
* `prefix` (default false) - Put included contents inside an object, instead
of loading them into the root. If no `key` is provided, one is automatically generated based on each files basename()
* `key` (default: <empty string>) - Key to load contents of include into. If
......@@ -273,7 +273,7 @@ object or an array.
* `priority` (default: 0) - specify priority for the include (see below).
* `duplicate` (default: 'append') - specify policy of duplicates resolving:
- `append` - default strategy, if we have new object of higher priority then it replaces old one, if we have new object with less priority it is ignored completely, and if we have two duplicate objects with the same priority then we have a multi-value key (implicit array)
- `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardeless of priorities)
- `merge` - if we have object or array, then new keys are merged inside, if we have a plain object then an implicit array is formed (regardless of priorities)
- `error` - create error on duplicate keys and stop parsing
- `rewrite` - always rewrite an old value with new one (ignoring priorities)
......@@ -320,7 +320,7 @@ Here are some rules for this syntax:
* Multiline terminator must start just after `<<` symbols and it must consist of capital letters only (e.g. `<<eof` or `<< EOF` won't work);
* Terminator must end with a single newline character (and no spaces are allowed between terminator and newline character);
* To finish multiline string you need to include a terminator string just after newline and followed by a newline (no spaces or other characters are allowed as well);
* The initial and the final newlines are not inserted to the resulting string, but you can still specify newlines at the begin and at the end of a value, for example:
* The initial and the final newlines are not inserted to the resulting string, but you can still specify newlines at the beginning and at the end of a value, for example:
```
key <<EOD
......@@ -347,7 +347,7 @@ UCL allows validation of objects. It uses the same schema that is used for json:
## Performance
Are UCL parser and emitter fast enough? Well, there are some numbers.
I got a 19Mb file that consist of ~700 thousands lines of json (obtained via
I got a 19Mb file that consist of ~700 thousand lines of json (obtained via
http://www.json-generator.com/). Then I checked jansson library that performs json
parsing and emitting and compared it with UCL. Here are results:
......@@ -377,6 +377,6 @@ You can do your own benchmarks by running `make check` in libucl top directory.
## Conclusion
UCL has clear design that should be very convenient for reading and writing. At the same time it is compatible with
JSON language and therefore can be used as a simple JSON parser. Macroes logic provides an ability to extend configuration
language (for example by including some lua code) and comments allows to disable or enable the parts of a configuration
JSON language and therefore can be used as a simple JSON parser. Macro logic provides an ability to extend configuration
language (for example by including some lua code) and comments allow to disable or enable the parts of a configuration
quickly.
{-# LANGUAGE ForeignFunctionInterface #-}
-- an example UCL FFI module:
-- uses the Object Model from Messagepack to emit
--
module Data.UCL ( unpack ) where
import Foreign.C
import Foreign.Ptr
import System.IO.Unsafe ( unsafePerformIO )
import qualified Data.Text as T
import qualified Data.Vector as V
import qualified Data.MessagePack as MSG
type ParserHandle = Ptr ()
type UCLObjectHandle = Ptr ()
type UCLIterHandle = Ptr ()
type UCLEmitterType = CInt
type ErrorString = String
foreign import ccall "ucl_parser_new" ucl_parser_new :: CInt -> ParserHandle
foreign import ccall "ucl_parser_add_string" ucl_parser_add_string :: ParserHandle -> CString -> CUInt -> IO Bool
foreign import ccall "ucl_parser_add_file" ucl_parser_add_file :: ParserHandle -> CString -> IO Bool
foreign import ccall "ucl_parser_get_object" ucl_parser_get_object :: ParserHandle -> UCLObjectHandle
foreign import ccall "ucl_parser_get_error" ucl_parser_get_error :: ParserHandle -> CString
foreign import ccall "ucl_object_iterate_new" ucl_object_iterate_new :: UCLObjectHandle -> UCLIterHandle
foreign import ccall "ucl_object_iterate_safe" ucl_object_iterate_safe :: UCLIterHandle -> Bool -> UCLObjectHandle
foreign import ccall "ucl_object_type" ucl_object_type :: UCLObjectHandle -> CUInt
foreign import ccall "ucl_object_key" ucl_object_key :: UCLObjectHandle -> CString
foreign import ccall "ucl_object_toint" ucl_object_toint :: UCLObjectHandle -> CInt
foreign import ccall "ucl_object_todouble" ucl_object_todouble :: UCLObjectHandle -> CDouble
foreign import ccall "ucl_object_tostring" ucl_object_tostring :: UCLObjectHandle -> CString
foreign import ccall "ucl_object_toboolean" ucl_object_toboolean :: UCLObjectHandle -> Bool
foreign import ccall "ucl_object_emit" ucl_object_emit :: UCLObjectHandle -> UCLEmitterType -> CString
foreign import ccall "ucl_object_emit_len" ucl_object_emit_len :: UCLObjectHandle -> UCLEmitterType -> Ptr CSize -> IO CString
type UCL_TYPE = CUInt
ucl_OBJECT :: UCL_TYPE
ucl_OBJECT = 0
ucl_ARRAY :: UCL_TYPE
ucl_ARRAY = 1
ucl_INT :: UCL_TYPE
ucl_INT = 2
ucl_FLOAT :: UCL_TYPE
ucl_FLOAT = 3
ucl_STRING :: UCL_TYPE
ucl_STRING = 4
ucl_BOOLEAN :: UCL_TYPE
ucl_BOOLEAN = 5
ucl_TIME :: UCL_TYPE
ucl_TIME = 6
ucl_USERDATA :: UCL_TYPE
ucl_USERDATA = 7
ucl_NULL :: UCL_TYPE
ucl_NULL = 8
ucl_emit_json :: UCLEmitterType
ucl_emit_json = 0
ucl_emit_json_compact :: UCLEmitterType
ucl_emit_json_compact = 1 :: UCLEmitterType
ucl_emit_msgpack :: UCLEmitterType
ucl_emit_msgpack = 4 :: UCLEmitterType
ucl_parser_parse_string_pure :: String -> Either UCLObjectHandle ErrorString
ucl_parser_parse_string_pure s = unsafePerformIO $ do
cs <- newCString s
let p = ucl_parser_new 0x4
didParse <- ucl_parser_add_string p cs (toEnum $ length s)
if didParse
then return $ Left $ ucl_parser_get_object p
else Right <$> peekCString ( ucl_parser_get_error p)
ucl_parser_add_file_pure :: String -> Either UCLObjectHandle ErrorString
ucl_parser_add_file_pure s = unsafePerformIO $ do
cs <- newCString s
let p = ucl_parser_new 0x4
didParse <- ucl_parser_add_file p cs
if didParse
then return $ Left $ ucl_parser_get_object p
else Right <$> peekCString ( ucl_parser_get_error p)
unpack :: MSG.MessagePack a => String -> Either a ErrorString
unpack s = case ucl_parser_parse_string_pure s of
(Right err) -> Right err
(Left obj) -> case MSG.fromObject (ucl_to_msgpack_object obj) of
Nothing -> Right "MessagePack fromObject Error"
(Just a) -> Left a
ucl_to_msgpack_object :: UCLObjectHandle -> MSG.Object
ucl_to_msgpack_object o = toMsgPackObj (ucl_object_type o) o
where
toMsgPackObj n obj
|n==ucl_OBJECT = MSG.ObjectMap $ uclObjectToVector obj
|n==ucl_ARRAY = MSG.ObjectArray undefined
|n==ucl_INT = MSG.ObjectInt $ fromEnum $ ucl_object_toint obj
|n==ucl_FLOAT = MSG.ObjectDouble $ realToFrac $ ucl_object_todouble obj
|n==ucl_STRING = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_tostring obj
|n==ucl_BOOLEAN = MSG.ObjectBool $ ucl_object_toboolean obj
|n==ucl_TIME = error "time undefined"
|n==ucl_USERDATA = error "userdata undefined"
|n==ucl_NULL = error "null undefined"
|otherwise = error "\"Unknown Type\" Error"
uclObjectToVector :: UCLObjectHandle -> V.Vector (MSG.Object,MSG.Object)
uclObjectToVector o = iterateObject (ucl_object_iterate_safe iter True ) iter V.empty
where
iter = ucl_object_iterate_new o
iterateObject obj it vec = if ucl_object_type obj == ucl_NULL
then vec
else iterateObject (ucl_object_iterate_safe it True) it (V.snoc vec ( getUclKey obj , ucl_to_msgpack_object obj))
getUclKey obj = MSG.ObjectStr $ T.pack $ unsafePerformIO $ peekCString $ ucl_object_key obj
uclArrayToVector :: UCLObjectHandle -> V.Vector MSG.Object
uclArrayToVector o = iterateArray (ucl_object_iterate_safe iter True ) iter V.empty
where
iter = ucl_object_iterate_new o
iterateArray obj it vec = if ucl_object_type obj == ucl_NULL
then vec
else iterateArray (ucl_object_iterate_safe it True) it (V.snoc vec (ucl_to_msgpack_object obj))
......@@ -24,6 +24,9 @@
#pragma once
#include <string>
#include <vector>
#include <map>
#include <set>
#include <memory>
#include <iostream>
......@@ -100,6 +103,68 @@ class Ucl final {
return func;
};
static bool ucl_variable_getter(const unsigned char *data, size_t len,
unsigned char ** /*replace*/, size_t * /*replace_len*/, bool *need_free, void* ud)
{
*need_free = false;
auto vars = reinterpret_cast<std::set<std::string> *>(ud);
if (vars && data && len != 0) {
vars->emplace (data, data + len);
}
return false;
}
static bool ucl_variable_replacer (const unsigned char *data, size_t len,
unsigned char **replace, size_t *replace_len, bool *need_free, void* ud)
{
*need_free = false;
auto replacer = reinterpret_cast<variable_replacer *>(ud);
if (!replacer) {
return false;
}
std::string var_name (data, data + len);
if (!replacer->is_variable (var_name)) {
return false;
}
std::string var_value = replacer->replace (var_name);
if (var_value.empty ()) {
return false;
}
*replace = (unsigned char *)UCL_ALLOC (var_value.size ());
memcpy (*replace, var_value.data (), var_value.size ());
*replace_len = var_value.size ();
*need_free = true;
return true;
}
template <typename C, typename P>
static Ucl parse_with_strategy_function (C config_func, P parse_func, std::string &err)
{
auto parser = ucl_parser_new (UCL_PARSER_DEFAULT);
config_func (parser);
if (!parse_func (parser)) {
err.assign (ucl_parser_get_error (parser));
ucl_parser_free (parser);
return nullptr;
}
auto obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
// Obj will handle ownership
return Ucl (obj);
}
std::unique_ptr<ucl_object_t, ucl_deleter> obj;
public:
......@@ -117,9 +182,9 @@ class Ucl final {
const_iterator(const Ucl &obj) {
it = std::shared_ptr<void>(ucl_object_iterate_new (obj.obj.get()),
ucl_iter_deleter());
ucl_iter_deleter());
cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true)));
if (!*cur) {
if (cur->type() == UCL_NULL) {
it.reset ();
cur.reset ();
}
......@@ -153,7 +218,7 @@ class Ucl final {
cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true)));
}
if (!*cur) {
if (cur && cur->type() == UCL_NULL) {
it.reset ();
cur.reset ();
}
......@@ -171,6 +236,17 @@ class Ucl final {
}
};
struct variable_replacer {
virtual ~variable_replacer() {}
virtual bool is_variable (const std::string &str) const
{
return !str.empty ();
}
virtual std::string replace (const std::string &var) const = 0;
};
// We grab ownership if get non-const ucl_object_t
Ucl(ucl_object_t *other) {
obj.reset (other);
......@@ -211,20 +287,20 @@ class Ucl final {
obj.reset (ucl_object_fromstring_common (value.data (), value.size (),
UCL_STRING_RAW));
}
Ucl(const char * value) {
Ucl(const char *value) {
obj.reset (ucl_object_fromstring_common (value, 0, UCL_STRING_RAW));
}
// Implicit constructor: anything with a to_json() function.
template <class T, class = decltype(&T::to_ucl)>
Ucl(const T & t) : Ucl(t.to_ucl()) {}
Ucl(const T &t) : Ucl(t.to_ucl()) {}
// Implicit constructor: map-like objects (std::map, std::unordered_map, etc)
template <class M, typename std::enable_if<
std::is_constructible<std::string, typename M::key_type>::value
&& std::is_constructible<Ucl, typename M::mapped_type>::value,
int>::type = 0>
Ucl(const M & m) {
Ucl(const M &m) {
obj.reset (ucl_object_typed_new (UCL_OBJECT));
auto cobj = obj.get ();
......@@ -238,7 +314,7 @@ class Ucl final {
template <class V, typename std::enable_if<
std::is_constructible<Ucl, typename V::value_type>::value,
int>::type = 0>
Ucl(const V & v) {
Ucl(const V &v) {
obj.reset (ucl_object_typed_new (UCL_ARRAY));
auto cobj = obj.get ();
......@@ -356,46 +432,138 @@ class Ucl final {
return out;
}
static Ucl parse (const std::string & in, std::string & err)
static Ucl parse (const std::string &in, std::string &err)
{
auto parser = ucl_parser_new (UCL_PARSER_DEFAULT);
return parse (in, std::map<std::string, std::string>(), err);
}
if (!ucl_parser_add_chunk (parser, (const unsigned char *)in.data (),
in.size ())) {
err.assign (ucl_parser_get_error (parser));
ucl_parser_free (parser);
static Ucl parse (const std::string &in, const std::map<std::string, std::string> &vars, std::string &err)
{
auto config_func = [&vars] (ucl_parser *parser) {
for (const auto & item : vars) {
ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ());
}
};
auto parse_func = [&in] (ucl_parser *parser) {
return ucl_parser_add_chunk (parser, (unsigned char *)in.data (), in.size ());
};
return parse_with_strategy_function (config_func, parse_func, err);
}
static Ucl parse (const std::string &in, const variable_replacer &replacer, std::string &err)
{
auto config_func = [&replacer] (ucl_parser *parser) {
ucl_parser_set_variables_handler (parser, ucl_variable_replacer,
&const_cast<variable_replacer &>(replacer));
};
auto parse_func = [&in] (ucl_parser *parser) {
return ucl_parser_add_chunk (parser, (unsigned char *) in.data (), in.size ());
};
return parse_with_strategy_function (config_func, parse_func, err);
}
static Ucl parse (const char *in, std::string &err)
{
return parse (in, std::map<std::string, std::string>(), err);
}
static Ucl parse (const char *in, const std::map<std::string, std::string> &vars, std::string &err)
{
if (!in) {
err = "null input";
return nullptr;
}
return parse (std::string (in), vars, err);
}
auto obj = ucl_parser_get_object (parser);
static Ucl parse (const char *in, const variable_replacer &replacer, std::string &err)
{
if (!in) {
err = "null input";
return nullptr;
}
return parse (std::string(in), replacer, err);
}
static Ucl parse_from_file (const std::string &filename, std::string &err)
{
return parse_from_file (filename, std::map<std::string, std::string>(), err);
}
static Ucl parse_from_file (const std::string &filename, const std::map<std::string, std::string> &vars, std::string &err)
{
auto config_func = [&vars] (ucl_parser *parser) {
for (const auto & item : vars) {
ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ());
}
};
auto parse_func = [&filename] (ucl_parser *parser) {
return ucl_parser_add_file (parser, filename.c_str ());
};
return parse_with_strategy_function (config_func, parse_func, err);
}
static Ucl parse_from_file (const std::string &filename, const variable_replacer &replacer, std::string &err)
{
auto config_func = [&replacer] (ucl_parser *parser) {
ucl_parser_set_variables_handler (parser, ucl_variable_replacer,
&const_cast<variable_replacer &>(replacer));
};
auto parse_func = [&filename] (ucl_parser *parser) {
return ucl_parser_add_file (parser, filename.c_str ());
};
return parse_with_strategy_function (config_func, parse_func, err);
}
static std::vector<std::string> find_variable (const std::string &in)
{
auto parser = ucl_parser_new (UCL_PARSER_DEFAULT);
std::set<std::string> vars;
ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars);
ucl_parser_add_chunk (parser, (const unsigned char *)in.data (), in.size ());
ucl_parser_free (parser);
// Obj will handle ownership
return Ucl (obj);
std::vector<std::string> result;
std::move (vars.begin (), vars.end (), std::back_inserter (result));
return result;
}
static Ucl parse (const char * in, std::string & err)
static std::vector<std::string> find_variable (const char *in)
{
if (in) {
return parse (std::string(in), err);
} else {
err = "null input";
return nullptr;
if (!in) {
return std::vector<std::string>();
}
return find_variable (std::string (in));
}
static Ucl parse (std::istream &ifs, std::string &err)
static std::vector<std::string> find_variable_from_file (const std::string &filename)
{
return Ucl::parse (std::string(std::istreambuf_iterator<char>(ifs),
std::istreambuf_iterator<char>()), err);
auto parser = ucl_parser_new (UCL_PARSER_DEFAULT);
std::set<std::string> vars;
ucl_parser_set_variables_handler (parser, ucl_variable_getter, &vars);
ucl_parser_add_file (parser, filename.c_str ());
ucl_parser_free (parser);
std::vector<std::string> result;
std::move (vars.begin (), vars.end (), std::back_inserter (result));
return std::move (result);
}
Ucl& operator= (Ucl rhs)
{
obj.swap (rhs.obj);
return *this;
}
Ucl& operator= (Ucl rhs)
{
obj.swap (rhs.obj);
return *this;
}
bool operator== (const Ucl &rhs) const
{
......
......@@ -1016,7 +1016,6 @@ UCL_EXTERN bool ucl_parser_add_string_priority (struct ucl_parser *parser,
* Load and add data from a file
* @param parser parser structure
* @param filename the name of file
* @param err if *err is NULL it is set to parser error
* @return true if chunk has been added and false in case of error
*/
UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser,
......@@ -1026,7 +1025,6 @@ UCL_EXTERN bool ucl_parser_add_file (struct ucl_parser *parser,
* Load and add data from a file
* @param parser parser structure
* @param filename the name of file