Update libucl to git version 8d3b186

ChangeLog.md

@@ -20,3 +20,15 @@
 ### Libucl 0.6.1
 
 - Various utilities fixes
+
+### Libucl 0.7.0
+
+- Move to klib library from uthash to reduce memory overhead and increase performance
+
+### Libucl 0.7.1
+
+- Added safe iterators API
+
+### Libucl 0.7.2
+
+- Fixed serious bugs in schema and arrays iteration

Makefile.am

 ACLOCAL_AMFLAGS = -I m4
-EXTRA_DIST = uthash README.md
+EXTRA_DIST = uthash klib README.md
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libucl.pc

README.md

 # LIBUCL
 
-[![Build Status](https://travis-ci.org/vstakhov/libucl.svg?branch=master)](https://travis-ci.org/vstakhov/libucl)
+[![Build Status](https://travis-ci.org/vstakhov/libucl.svg?branch=master)](https://travis-ci.org/vstakhov/libucl)[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138)
 
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
@@ -156,10 +156,10 @@ is converted to the following object:
 ```nginx
 section {
 	blah {
-			key = value;
+		key = value;
 	}
 	foo {
-			key = value;
+		key = value;
 	}
 }
 ```
@@ -177,9 +177,9 @@ is presented as:
 ```nginx    
 section {
 	blah {
-			foo {
-					key = value;
-			}
+		foo {
+			key = value;
+		}
 	}
 }
 ```
@@ -219,8 +219,8 @@ UCL supports external macros both multiline and single line ones:
 ```nginx
 .macro "sometext";
 .macro {
-     Some long text
-     ....
+    Some long text
+    ....
 };
 ```
 

cmake/CMakeLists.txt

@@ -82,6 +82,7 @@ ENDIF(ENABLE_URL_SIGN MATCHES "ON")
 INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../src")
 INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../include")
 INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../uthash")
+INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../klib")
 
 SET(UCLSRC            ../src/ucl_util.c
                       ../src/ucl_parser.c

configure.ac

 m4_define([maj_ver], [0])
-m4_define([med_ver], [6])
-m4_define([min_ver], [1])
-m4_define([so_version], [3:0:1])
+m4_define([med_ver], [7])
+m4_define([min_ver], [2])
+m4_define([so_version], [5:0:1])
 m4_define([ucl_version], [maj_ver.med_ver.min_ver])
 
 AC_INIT([libucl],[ucl_version],[https://github.com/vstakhov/libucl],[libucl])

doc/Makefile.am

@@ -4,5 +4,6 @@ dist_man_MANS = libucl.3
 
 gen-man: @PANDOC@
 	tail -n +$$(grep -n '# Synopsis' api.md | cut -d':' -f1) api.md | \
-	cat pandoc.template - | sed -e 's/^# \(.*\)/# \U\1/' | \
+	cat pandoc.template - | sed -e 's/^# \(.*\)/# \U\1/' \ 
+	-e "s/%%date%%/$$(LANG=C date +'%d %B, %Y')/" | \
 	@PANDOC@ -s -f markdown -t man -o libucl.3 
\ No newline at end of file

doc/api.md

@@ -377,7 +377,9 @@ If parsing operations fail then the resulting UCL object will be a `UCL_STRING`.
 
 # Iteration functions
 
-Iteration are used to iterate over UCL compound types: arrays and objects. Moreover, iterations could be performed over the keys with multiple values (implicit arrays). To iterate over an object, an array or a key with multiple values there is a function `ucl_iterate_object`.
+Iteration are used to iterate over UCL compound types: arrays and objects. Moreover, iterations could be performed over the keys with multiple values (implicit arrays).
+There are two types of iterators API: old and unsafe one via `ucl_iterate_object` and the proposed interface of safe iterators.
+
 
 ## ucl_iterate_object
 ~~~C
@@ -402,6 +404,60 @@ while ((obj = ucl_iterate_object (top, &it, true))) {
 }
 ~~~
 
+## Safe iterators API
+
+Safe iterators are defined to clarify iterating over UCL objects and simplify flattening of UCL objects in non-trivial cases.
+For example, if there is an implicit array that contains another array and a boolean value it is extremely unclear how to iterate over
+such an object. Safe iterators are desinged to define two sorts of iteration:
+
+1. Iteration over complex objects with expanding all values
+2. Iteration over complex objects without expanding of values
+
+The following example demonstrates the difference between these two types of iteration:
+
+~~~
+key = 1;
+key = [2, 3, 4];
+
+Iteration with expansion:
+
+1, 2, 3, 4
+
+Iteration without expansion:
+
+1, [2, 3, 4]
+~~~
+
+UCL defines the following functions to manage safe iterators:
+
+- `ucl_object_iterate_new` - creates new safe iterator
+- `ucl_object_iterate_reset` - resets iterator to a new object
+- `ucl_object_iterate_safe` - safely iterate the object inside iterator
+- `ucl_object_iterate_free` - free memory associated with the safe iterator
+
+Please note that unlike unsafe iterators, safe iterators *must* be explicitly initialized and freed.
+An assert is likely generated if you use uninitialized or `NULL` iterator in all safe iterators functions.
+
+~~~C
+ucl_object_iter_t it;
+const ucl_object_t *cur;
+
+it = ucl_object_iterate_new (obj);
+
+while ((cur = ucl_object_iterate_safe (it, true)) != NULL) {
+	/* Do something */
+}
+
+/* Switch to another object */
+it = ucl_object_iterate_reset (it, another_obj);
+
+while ((cur = ucl_object_iterate_safe (it, true)) != NULL) {
+	/* Do something else */
+}
+
+ucl_object_iterate_free (it);
+~~~
+
 # Validation functions
 
 Currently, there is only one validation function called `ucl_object_validate`. It performs validation of object using the specified schema. This function is defined as following:

doc/libucl.3

-.TH "LIBUCL" "3" "July 26, 2014" "Libucl manual" ""
+.TH "LIBUCL" "3" "27 December, 2014" "Libucl manual" ""
 .SH NAME
 .PP
 \f[B]ucl_parser_new\f[], \f[B]ucl_parser_register_macro\f[],
@@ -528,8 +528,9 @@ Iteration are used to iterate over UCL compound types: arrays and
 objects.
 Moreover, iterations could be performed over the keys with multiple
 values (implicit arrays).
-To iterate over an object, an array or a key with multiple values there
-is a function \f[C]ucl_iterate_object\f[].
+There are two types of iterators API: old and unsafe one via
+\f[C]ucl_iterate_object\f[] and the proposed interface of safe
+iterators.
 .SS ucl_iterate_object
 .IP
 .nf
@@ -578,6 +579,75 @@ while\ ((obj\ =\ ucl_iterate_object\ (top,\ &it,\ true)))\ {
 }
 \f[]
 .fi
+.SS Safe iterators API
+.PP
+Safe iterators are defined to clarify iterating over UCL objects and
+simplify flattening of UCL objects in non\-trivial cases.
+For example, if there is an implicit array that contains another array
+and a boolean value it is extremely unclear how to iterate over such an
+object.
+Safe iterators are desinged to define two sorts of iteration:
+.IP "1." 3
+Iteration over complex objects with expanding all values
+.IP "2." 3
+Iteration over complex objects without expanding of values
+.PP
+The following example demonstrates the difference between these two
+types of iteration:
+.IP
+.nf
+\f[C]
+key\ =\ 1;
+key\ =\ [2,\ 3,\ 4];
+
+Iteration\ with\ expansion:
+
+1,\ 2,\ 3,\ 4
+
+Iteration\ without\ expansion:
+
+1,\ [2,\ 3,\ 4]
+\f[]
+.fi
+.PP
+UCL defines the following functions to manage safe iterators:
+.IP \[bu] 2
+\f[C]ucl_object_iterate_new\f[] \- creates new safe iterator
+.IP \[bu] 2
+\f[C]ucl_object_iterate_reset\f[] \- resets iterator to a new object
+.IP \[bu] 2
+\f[C]ucl_object_iterate_safe\f[] \- safely iterate the object inside
+iterator
+.IP \[bu] 2
+\f[C]ucl_object_iterate_free\f[] \- free memory associated with the safe
+iterator
+.PP
+Please note that unlike unsafe iterators, safe iterators \f[I]must\f[]
+be explicitly initialized and freed.
+An assert is likely generated if you use uninitialized or \f[C]NULL\f[]
+iterator in all safe iterators functions.
+.IP
+.nf
+\f[C]
+ucl_object_iter_t\ it;
+const\ ucl_object_t\ *cur;
+
+it\ =\ ucl_object_iterate_new\ (obj);
+
+while\ ((cur\ =\ ucl_object_iterate_safe\ (it,\ true))\ !=\ NULL)\ {
+\ \ \ \ /*\ Do\ something\ */
+}
+
+/*\ Switch\ to\ another\ object\ */
+it\ =\ ucl_object_iterate_reset\ (it,\ another_obj);
+
+while\ ((cur\ =\ ucl_object_iterate_safe\ (it,\ true))\ !=\ NULL)\ {
+\ \ \ \ /*\ Do\ something\ else\ */
+}
+
+ucl_object_iterate_free\ (it);
+\f[]
+.fi
 .SH VALIDATION FUNCTIONS
 .PP
 Currently, there is only one validation function called

doc/pandoc.template

 % LIBUCL(3) Libucl manual
 % Vsevolod Stakhov <vsevolod@highsecure.ru>
-% July 26, 2014
+% %%date%%
 
 # Name
 

include/ucl.h

@@ -192,7 +192,7 @@ typedef struct ucl_object_s {
 		int64_t iv;							/**< Int value of an object */
 		const char *sv;					/**< String value of an object */
 		double dv;							/**< Double value of an object */
-		struct ucl_object_s *av;			/**< Array					*/
+		void *av;							/**< Array					*/
 		void *ov;							/**< Object					*/
 		void* ud;							/**< Opaque user data		*/
 	} value;
@@ -715,6 +715,37 @@ typedef void* ucl_object_iter_t;
  */
 UCL_EXTERN const ucl_object_t* ucl_iterate_object (const ucl_object_t *obj,
 		ucl_object_iter_t *iter, bool expand_values);
+
+/**
+ * Create new safe iterator for the specified object
+ * @param obj object to iterate
+ * @return new iterator object that should be used with safe iterators API only
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_new (const ucl_object_t *obj)
+	UCL_WARN_UNUSED_RESULT;
+/**
+ * Reset initialized iterator to a new object
+ * @param obj new object to iterate
+ * @return modified iterator object
+ */
+UCL_EXTERN ucl_object_iter_t ucl_object_iterate_reset (ucl_object_iter_t it,
+		const ucl_object_t *obj);
+
+/**
+ * Get the next object from the `obj`. This fucntion iterates over arrays, objects
+ * and implicit arrays
+ * @param iter safe iterator
+ * @return the next object in sequence
+ */
+UCL_EXTERN const ucl_object_t* ucl_object_iterate_safe (ucl_object_iter_t iter,
+		bool expand_values);
+
+/**
+ * Free memory associated with the safe iterator
+ * @param it safe iterator object
+ */
+UCL_EXTERN void ucl_object_iterate_free (ucl_object_iter_t it);
+
 /** @} */
 
 
@@ -854,6 +885,13 @@ UCL_EXTERN ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser);
  * @param parser parser object
  */
 UCL_EXTERN const char *ucl_parser_get_error(struct ucl_parser *parser);
+
+/**
+ * Clear the error in the parser
+ * @param parser parser object
+ */
+UCL_EXTERN void ucl_parser_clear_error(struct ucl_parser *parser);
+
 /**
  * Free ucl parser object
  * @param parser parser object

klib/kvec.h

+/* The MIT License
+
+   Copyright (c) 2008, by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "kvec.h"
+int main() {
+	kvec_t(int) array;
+	kv_init(array);
+	kv_push(int, array, 10); // append
+	kv_a(int, array, 20) = 5; // dynamic
+	kv_A(array, 20) = 4; // static
+	kv_destroy(array);
+	return 0;
+}
+*/
+
+/*
+  2008-09-22 (0.1.0):
+
+	* The initial version.
+
+*/
+
+#ifndef AC_KVEC_H
+#define AC_KVEC_H
+
+#include <stdlib.h>
+
+#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0)
+#define kv_destroy(v) free((v).a)
+#define kv_A(v, i) ((v).a[(i)])
+#define kv_pop(v) ((v).a[--(v).n])
+#define kv_size(v) ((v).n)
+#define kv_max(v) ((v).m)
+
+#define kv_resize(type, v, s)  ((v).m = (s), (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+#define kv_grow_factor 1.5
+#define kv_grow(type, v)  ((v).m = ((v).m > 1 ? (v).m * kv_grow_factor : 2), \
+		(v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_copy(type, v1, v0) do {											\
+		if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n);					\
+		(v1).n = (v0).n;													\
+		memcpy((v1).a, (v0).a, sizeof(type) * (v0).n);						\
+	} while (0)																\
+
+#define kv_push(type, v, x) do {											\
+		if ((v).n == (v).m) {												\
+			kv_grow(type, v);												\
+		}																	\
+		(v).a[(v).n++] = (x);												\
+	} while (0)
+
+#define kv_prepend(type, v, x) do {											\
+	if ((v).n == (v).m) {													\
+		kv_grow(type, v);													\
+	}																		\
+	memmove((v).a + 1, (v).a, sizeof(type) * (v).n);							\
+	(v).a[0] = (x);															\
+	(v).n ++;																\
+} while (0)
+
+#define kv_concat(type, v1, v0) do {										\
+	if ((v1).m < (v0).n + (v1).n) kv_resize(type, v1, (v0).n + (v1).n);		\
+		memcpy((v1).a + (v1).n, (v0).a, sizeof(type) * ((v0).n + (v1).n));	\
+		(v1).n = (v0).n + (v1).n;											\
+	} while (0)
+
+#define kv_del(type, v, i) do {												\
+	if ((i) < (v).n) {														\
+		memmove((v).a + (i), (v).a + ((i) + 1), sizeof(type) * ((v).n - (i) - 1)); \
+		(v).n --;															\
+	}																		\
+} while (0)
+
+#endif

m4/.gitignore

-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

src/Makefile.am

 libucl_common_cflags=	-I$(top_srcdir)/src \
 			-I$(top_srcdir)/include \
 			-I$(top_srcdir)/uthash \
+			-I$(top_srcdir)/klib \
 			-Wall -W -Wno-unused-parameter -Wno-pointer-sign
 lib_LTLIBRARIES=	libucl.la
 libucl_la_SOURCES=	ucl_emitter.c \

src/ucl_emitter.c

@@ -250,6 +250,7 @@ ucl_emitter_common_start_array (struct ucl_emitter_context *ctx,
 		const ucl_object_t *obj, bool print_key, bool compact)
 {
 	const ucl_object_t *cur;
+	ucl_object_iter_t iter = NULL;
 	const struct ucl_emitter_functions *func = ctx->func;
 	bool first = true;
 
@@ -266,18 +267,22 @@ ucl_emitter_common_start_array (struct ucl_emitter_context *ctx,
 
 	if (obj->type == UCL_ARRAY) {
 		/* explicit array */
-		cur = obj->value.av;
+		while ((cur = ucl_iterate_object (obj, &iter, true)) != NULL) {
+			ucl_emitter_common_elt (ctx, cur, first, false, compact);
+			first = false;
+		}
 	}
 	else {
 		/* implicit array */
 		cur = obj;
+		while (cur) {
+			ucl_emitter_common_elt (ctx, cur, first, false, compact);
+			first = false;
+			cur = cur->next;
+		}
 	}
 
-	while (cur) {
-		ucl_emitter_common_elt (ctx, cur, first, false, compact);
-		first = false;
-		cur = cur->next;
-	}
+
 }
 
 /**

src/ucl_emitter_utils.c

@@ -289,6 +289,7 @@ ucl_fd_append_character (unsigned char c, size_t len, void *ud)
 		else {
 			memset (buf, c, len);
 			if (write (fd, buf, len) == -1) {
+				free(buf);
 				return -1;
 			}
 			free (buf);

src/ucl_hash.c

@@ -23,119 +23,331 @@
 
 #include "ucl_internal.h"
 #include "ucl_hash.h"
-#include "utlist.h"
+#include "khash.h"
+#include "kvec.h"
+
+struct ucl_hash_elt {
+	const ucl_object_t *obj;
+	size_t ar_idx;
+};
+
+struct ucl_hash_struct {
+	void *hash;
+	kvec_t(const ucl_object_t *) ar;
+	bool caseless;
+};
+
+static inline uint32_t
+ucl_hash_func (const ucl_object_t *o)
+{
+	return XXH32 (o->key, o->keylen, 0xdeadbeef);
+}
+
+static inline int
+ucl_hash_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+	if (k1->keylen == k2->keylen) {
+		return strncmp (k1->key, k2->key, k1->keylen) == 0;
+	}
+
+	return 0;
+}
+
+KHASH_INIT (ucl_hash_node, const ucl_object_t *, struct ucl_hash_elt, 1,
+		ucl_hash_func, ucl_hash_equal)
+
+static inline uint32_t
+ucl_hash_caseless_func (const ucl_object_t *o)
+{
+	void *xxh = XXH32_init (0xdeadbeef);
+	char hash_buf[64], *c;
+	const char *p;
+	ssize_t remain = o->keylen;
+
+	p = o->key;
+	c = &hash_buf[0];
+
+	while (remain > 0) {
+		*c++ = tolower (*p++);
+
+		if (c - &hash_buf[0] == sizeof (hash_buf)) {
+			XXH32_update (xxh, hash_buf, sizeof (hash_buf));
+			c = &hash_buf[0];
+		}
+		remain --;
+	}
+
+	if (c - &hash_buf[0] != 0) {
+		XXH32_update (xxh, hash_buf, c - &hash_buf[0]);
+	}
+
+	return XXH32_digest (xxh);
+}
+
+static inline int
+ucl_hash_caseless_equal (const ucl_object_t *k1, const ucl_object_t *k2)
+{
+	if (k1->keylen == k2->keylen) {
+		return strncasecmp (k1->key, k2->key, k1->keylen) == 0;
+	}
+
+	return 0;
+}
+
+KHASH_INIT (ucl_hash_caseless_node, const ucl_object_t *, struct ucl_hash_elt, 1,
+		ucl_hash_caseless_func, ucl_hash_caseless_equal)
 
 ucl_hash_t*
-ucl_hash_create (void)
+ucl_hash_create (bool ignore_case)
 {
 	ucl_hash_t *new;
 
 	new = UCL_ALLOC (sizeof (ucl_hash_t));
 	if (new != NULL) {
-		new->buckets = NULL;
+		kv_init (new->ar);
+
+		new->caseless = ignore_case;
+		if (ignore_case) {
+			khash_t(ucl_hash_caseless_node) *h = kh_init (ucl_hash_caseless_node);
+			new->hash = (void *)h;
+		}
+		else {
+			khash_t(ucl_hash_node) *h = kh_init (ucl_hash_node);
+			new->hash = (void *)h;
+		}
 	}
 	return new;
 }
 
 void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func *func)
 {
-	ucl_hash_node_t *elt, *tmp;
-	const ucl_object_t *cur, *otmp;
-
-	HASH_ITER (hh, hashlin->buckets, elt, tmp) {
-		HASH_DELETE (hh, hashlin->buckets, elt);
-		if (func) {
-			DL_FOREACH_SAFE (elt->data, cur, otmp) {
-				/* Need to deconst here */
-				func (__DECONST (ucl_object_t *, cur));
+	const ucl_object_t *cur, *tmp;
+
+	if (hashlin == NULL) {
+		return;
+	}
+
+	if (func != NULL) {
+		/* Iterate over the hash first */
+		khash_t(ucl_hash_node) *h = (khash_t(ucl_hash_node) *)
+				hashlin->hash;
+		khiter_t k;
+
+		for (k = kh_begin (h); k != kh_end (h); ++k) {
+			if (kh_exist (h, k)) {
+				cur = (kh_value (h, k)).obj;
+				while (cur != NULL) {
+					tmp = cur->next;
+					func (__DECONST (ucl_object_t *, cur));
+					cur = tmp;
+				}
 			}
 		}
-		UCL_FREE (sizeof (ucl_hash_node_t), elt);
 	}
-	UCL_FREE (sizeof (ucl_hash_t), hashlin);
+