Commit 80fbb61f authored by bdrewery's avatar bdrewery
Browse files

Add a patch for memcached to ccache along with a slave devel/ccache-memcached port.

This patch is not safe for WITH_CCACHE_BUILD support yet as that causes all
ports to depend on devel/ccache.  Enabling that patch would then cause the
new devel/libmemcached dependency to require devel/ccache which is a cyclic
dependency.  The autoconf dependency also causes issues.

Add a devel/ccache-memcached slave port that would allow a user to use
the ccache+memcached package manually with ports without WITH_CCACHE_BUILD.

This patch comes from https://github.com/ccache/ccache/pull/58 and has been
an ongoing effort over a few years to be merged into the mainline of ccache.
Documenation for it can be found in the MANUAL file at:

  /usr/local/share/doc/ccache/MANUAL.txt

Sponsored by:	Dell EMC Isilon
parent c1f95d13
......@@ -234,6 +234,7 @@
SUBDIR += cbrowser
SUBDIR += cc65
SUBDIR += ccache
SUBDIR += ccache-memcached
SUBDIR += cccc
SUBDIR += ccdoc
SUBDIR += ccons
......
# $FreeBSD$
PKGNAMESUFFIX= -memcached
MASTERDIR= ${.CURDIR}/../ccache
OPTIONS_SLAVE= MEMCACHED
CONFLICTS_INSTALL= ccache-[0-9]*
.include "${MASTERDIR}/Makefile"
......@@ -13,26 +13,43 @@ COMMENT= Tool to minimize the compile time of C/C++ programs
LICENSE= GPLv3
CONFLICTS_INSTALL= ccache-memcached-[0-9]*
GNU_CONFIGURE= yes
HOWTO= ccache-howto-freebsd.txt
CCLINKDIR= libexec/ccache
SUB_FILES= ${HOWTO} world-ccache pkg-message ccache-update-links.sh
PORTDOCS= ccache-howto-freebsd.txt MANUAL.html
PORTDOCS= ccache-howto-freebsd.txt MANUAL.html MANUAL.txt
OPTIONS_DEFINE= CLANGLINK LLVMLINK STATIC DOCS TINDERBOX
OPTIONS_DEFINE= CLANGLINK LLVMLINK STATIC DOCS TINDERBOX MEMCACHED
OPTIONS_DEFAULT=CLANGLINK LLVMLINK
CLANGLINK_DESC= Create clang compiler links if clang is installed
LLVMLINK_DESC= Create llvm compiler links if llvm is installed
TINDERBOX_DESC= Create tarball for tinderbox usage
MEMCACHED_DESC= Build in experimental Memcached support
USES= compiler
MEMCACHED_EXTRA_PATCHES= ${FILESDIR}/extra-patch-memcached:-p1
MEMCACHED_CONFIGURE_ENABLE= memcached
MEMCACHED_USES= autoreconf pkgconfig
MEMCACHED_LIB_DEPENDS= libmemcached.so:databases/libmemcached
MEMCACHED_LDFLAGS= -L${LOCALBASE}/lib
MEMCACHED_CFLAGS= -I${LOCALBASE}/include
.if defined(WITH_CCACHE_BUILD) && empty(OPTIONS_SLAVE:MMEMCACHED)
# Don't allow autoreconf. We want no dependencies on this to keep
# WITH_CCACHE_BUILD working.
USES:= ${USES:Nautoreconf}
MEMCACHED_IGNORE= MEMCACHED cannot be combined with WITH_CCACHE_BUILD. Use devel/ccache-memcached
# XXX: This needs more testing with Poudriere before enabling. Also bsd.options.mk support.
#MEMCACHED_DEPENDS_ARGS+= NO_CCACHE=1
.endif
OPTIONS_SUB= yes
STATIC_LDFLAGS= -static
......@@ -93,6 +110,7 @@ do-install-TINDERBOX-on:
do-install-DOCS-on:
${MKDIR} ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKSRC}/MANUAL.html ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKSRC}/MANUAL.txt ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKDIR}/${HOWTO} ${STAGEDIR}${DOCSDIR}
.include <bsd.port.post.mk>
https://github.com/ccache/ccache/pull/58
Retrieved on February 13th 2017.
Changes to .travis.yml removed since it is not in the release image.
diff --git a/MANUAL.txt b/MANUAL.txt
index ab01886..c78bb6e 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -418,6 +418,20 @@ WRAPPERS>>.
The default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki,
Mi, Gi, Ti (binary). The default suffix is "G".
+*memcached_conf* (*CCACHE_MEMCACHED_CONF*)::
+
+ The memcached_conf option sets the memcached(3) configuration to use for
+ storing and getting cache values, if any. Example configuration:
++
+-------------------------------------------------------------------------------
+CCACHE_MEMCACHED_CONF=--SERVER=localhost:11211
+-------------------------------------------------------------------------------
+
+*memcached_only* (*CCACHE_MEMCACHED_ONLY*)::
+
+ Only store files in memcached, don't store them in the local filesystems.
+ The manifests (for direct mode) and stats are still being stored locally.
+
*path* (*CCACHE_PATH*)::
If set, ccache will search directories in this list when looking for the
@@ -451,6 +465,11 @@ WRAPPERS>>.
from the cache using the direct mode, not the preprocessor mode. See
documentation for *read_only* regarding using a read-only ccache directory.
+*read_only_memcached* (*CCACHE_READONLY_MEMCACHED* or *CCACHE_NOREADONLY_MEMCACHED*), see <<_boolean_values,Boolean values>> above)::
+
+ If true, ccache will attempt to get previously cached values from memcached,
+ but will not try to store any new values in memcached.
+
*recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see <<_boolean_values,Boolean values>> above)::
If true, ccache will not use any previously stored result. New results will
@@ -769,6 +788,29 @@ A tip is to set *temporary_dir* to a directory on the local host to avoid NFS
traffic for temporary files.
+Sharing a cache with memcached
+------------------------------
+
+When using the *memcached* (<http://memcached.org>) feature, the most recently
+used cache entries are also available from the configured memcached servers.
+
+The local cache directory will be searched first, but then it will still be
+possible to get cache hits (over the network) before having to run the
+compiler.
+
+Using a local *moxi* (memcached proxy) will enable multiple ccache invocations
+to share memcached connections and thus avoid some of the network overhead.
+
+It will also allow you to fine-tune connection timeouts and other settings. You
+can optionally replace your memcached servers with Couchbase servers.
+
+Example:
+
+-------------------------------------------------------------------------------
+moxi -z 11211=mc_server1:11211,mc_server2:11211
+-------------------------------------------------------------------------------
+
+
Using ccache with other compiler wrappers
-----------------------------------------
diff --git a/Makefile.in b/Makefile.in
index 5aee02d..08b3633 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -37,6 +37,7 @@ non_3pp_sources = \
lockfile.c \
manifest.c \
mdfour.c \
+ memccached.c \
stats.c \
unify.c \
util.c \
@@ -101,7 +102,7 @@ perf: ccache$(EXEEXT)
.PHONY: test
test: ccache$(EXEEXT) test/main$(EXEEXT)
test/main$(EXEEXT)
- CC='$(CC)' $(srcdir)/test.sh
+ CC='$(CC)' @ccache_memcached@$(srcdir)/test.sh
.PHONY: quicktest
quicktest: test/main$(EXEEXT)
diff --git a/ccache.c b/ccache.c
index 88e0ec5..12026c7 100644
--- a/ccache.c
+++ b/ccache.c
@@ -102,6 +102,9 @@ static char *output_dia = NULL;
// Split dwarf information (GCC 4.8 andup). Contains pathname if not NULL.
static char *output_dwo = NULL;
+// The cached key.
+static char *cached_key;
+
// Array for storing -arch options.
#define MAX_ARCH_ARGS 10
static size_t arch_args_size = 0;
@@ -123,6 +126,9 @@ static char *cached_stderr;
// (cachedir/a/b/cdef[...]-size.d).
static char *cached_dep;
+// The manifest key.
+static char *manifest_name;
+
// Full path to the file containing the coverage information
// (cachedir/a/b/cdef[...]-size.gcno).
static char *cached_cov;
@@ -239,6 +245,18 @@ static pid_t compiler_pid = 0;
// stored in the cache changes in a backwards-incompatible way.
static const char HASH_PREFIX[] = "3";
+static void from_fscache(enum fromcache_call_mode mode,
+ bool put_object_in_manifest);
+static void to_fscache(struct args *args);
+#ifdef HAVE_LIBMEMCACHED
+static void from_memcached(enum fromcache_call_mode mode,
+ bool put_object_in_manifest);
+static void to_memcached(struct args *args);
+#endif
+static void (*from_cache)(enum fromcache_call_mode mode,
+ bool put_object_in_manifest);
+static void (*to_cache)(struct args *args);
+
static void
add_prefix(struct args *args, char *prefix_command)
{
@@ -952,6 +970,28 @@ put_file_in_cache(const char *source, const char *dest)
stats_update_size(file_size(&st), 1);
}
+#ifdef HAVE_LIBMEMCACHED
+// Copy data to the cache.
+static void
+put_data_in_cache(void *data, size_t size, const char *dest)
+{
+ int ret;
+
+ assert(!conf->read_only);
+ assert(!conf->read_only_direct);
+
+ /* already compressed (in cache) */
+ ret = write_file(data, dest, size);
+ if (ret != 0) {
+ cc_log("Failed to write to %s: %s", dest, strerror(errno));
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ cc_log("Stored in cache: %zu bytes -> %s", size, dest);
+ stats_update_size(size, 1);
+}
+#endif
+
// Copy or link a file from the cache.
static void
get_file_from_cache(const char *source, const char *dest)
@@ -1006,6 +1046,11 @@ send_cached_stderr(void)
// Create or update the manifest file.
void update_manifest_file(void)
{
+#ifdef HAVE_LIBMEMCACHED
+ char *data;
+ size_t size;
+#endif
+
if (!conf->direct_mode
|| !included_files
|| conf->read_only
@@ -1023,6 +1068,14 @@ void update_manifest_file(void)
update_mtime(manifest_path);
if (x_stat(manifest_path, &st) == 0) {
stats_update_size(file_size(&st) - old_size, old_size == 0 ? 1 : 0);
+#if HAVE_LIBMEMCACHED
+ if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached &&
+ read_file(manifest_path, st.st_size, &data, &size)) {
+ cc_log("Storing %s in memcached", manifest_name);
+ memccached_raw_set(manifest_name, data, size);
+ free(data);
+ }
+#endif
}
} else {
cc_log("Failed to add object file hash to %s", manifest_path);
@@ -1031,8 +1084,12 @@ void update_manifest_file(void)
// Run the real compiler and put the result in cache.
static void
-to_cache(struct args *args)
+to_fscache(struct args *args)
{
+#ifdef HAVE_LIBMEMCACHED
+ char *data_obj, *data_stderr, *data_dia, *data_dep;
+ size_t size_obj, size_stderr, size_dia, size_dep;
+#endif
char *tmp_stdout = format("%s.tmp.stdout", cached_obj);
int tmp_stdout_fd = create_tmp_fd(&tmp_stdout);
char *tmp_stderr = format("%s.tmp.stderr", cached_obj);
@@ -1288,6 +1345,40 @@ to_cache(struct args *args)
}
}
+#ifdef HAVE_LIBMEMCACHED
+ if (strlen(conf->memcached_conf) > 0 && !conf->read_only_memcached &&
+ !using_split_dwarf && /* no support for the dwo files just yet */
+ !generating_coverage) { /* coverage refers to local paths anyway */
+ cc_log("Storing %s in memcached", cached_key);
+ if (!read_file(cached_obj, 0, &data_obj, &size_obj)) {
+ data_obj = NULL;
+ size_obj = 0;
+ }
+ if (!read_file(cached_stderr, 0, &data_stderr, &size_stderr)) {
+ data_stderr = NULL;
+ size_stderr = 0;
+ }
+ if (!read_file(cached_dia, 0, &data_dia, &size_dia)) {
+ data_dia = NULL;
+ size_dia = 0;
+ }
+ if (!read_file(cached_dep, 0, &data_dep, &size_dep)) {
+ data_dep = NULL;
+ size_dep = 0;
+ }
+
+ if (data_obj) {
+ memccached_set(cached_key,
+ data_obj, data_stderr, data_dia, data_dep,
+ size_obj, size_stderr, size_dia, size_dep);
+ }
+
+ free(data_obj);
+ free(data_stderr);
+ free(data_dia);
+ free(data_dep);
+ }
+#endif
// Everything OK.
send_cached_stderr();
update_manifest_file();
@@ -1298,6 +1389,226 @@ to_cache(struct args *args)
free(tmp_dwo);
}
+#ifdef HAVE_LIBMEMCACHED
+// Run the real compiler and put the result in cache.
+static void
+to_memcached(struct args *args)
+{
+ const char *tmp_dir = temp_dir();
+ char *tmp_stdout, *tmp_stderr;
+ char *stderr_d, *obj_d, *dia_d = NULL, *dep_d = NULL;
+ size_t stderr_l = 0, obj_l = 0, dia_l = 0, dep_l = 0;
+ struct stat st;
+ int status, tmp_stdout_fd, tmp_stderr_fd;
+
+ tmp_stdout = format("%s/%s.tmp.stdout.%s", tmp_dir, cached_obj, tmp_string());
+ tmp_stdout_fd = create_tmp_fd(&tmp_stdout);
+ tmp_stderr = format("%s/%s.tmp.stderr.%s", tmp_dir, cached_obj, tmp_string());
+ tmp_stderr_fd = create_tmp_fd(&tmp_stderr);
+
+ if (generating_coverage) {
+ cc_log("No memcached support for coverage yet");
+ failed();
+ }
+ if (using_split_dwarf) {
+ cc_log("No memcached support for split dwarf yet");
+ failed();
+ }
+
+ if (create_parent_dirs(tmp_stdout) != 0) {
+ fatal("Failed to create parent directory for %s: %s",
+ tmp_stdout, strerror(errno));
+ }
+
+ args_add(args, "-o");
+ args_add(args, output_obj);
+
+ if (output_dia) {
+ args_add(args, "--serialize-diagnostics");
+ args_add(args, output_dia);
+ }
+
+ /* Turn off DEPENDENCIES_OUTPUT when running cc1, because
+ * otherwise it will emit a line like
+ *
+ * tmp.stdout.vexed.732.o: /home/mbp/.ccache/tmp.stdout.vexed.732.i
+ */
+ x_unsetenv("DEPENDENCIES_OUTPUT");
+
+ if (conf->run_second_cpp) {
+ args_add(args, input_file);
+ } else {
+ args_add(args, i_tmpfile);
+ }
+
+ cc_log("Running real compiler");
+ status = execute(args->argv, tmp_stdout_fd, tmp_stderr_fd, &compiler_pid);
+ args_pop(args, 3);
+
+ if (x_stat(tmp_stdout, &st) != 0) {
+ /* The stdout file was removed - cleanup in progress? Better bail out. */
+ stats_update(STATS_MISSING);
+ tmp_unlink(tmp_stdout);
+ tmp_unlink(tmp_stderr);
+ failed();
+ }
+ if (st.st_size != 0) {
+ cc_log("Compiler produced stdout");
+ stats_update(STATS_STDOUT);
+ tmp_unlink(tmp_stdout);
+ tmp_unlink(tmp_stderr);
+ failed();
+ }
+ tmp_unlink(tmp_stdout);
+
+ /*
+ * Merge stderr from the preprocessor (if any) and stderr from the real
+ * compiler into tmp_stderr.
+ */
+ if (cpp_stderr) {
+ int fd_cpp_stderr;
+ int fd_real_stderr;
+ int fd_result;
+ char *tmp_stderr2;
+
+ tmp_stderr2 = format("%s.2", tmp_stderr);
+ if (x_rename(tmp_stderr, tmp_stderr2)) {
+ cc_log("Failed to rename %s to %s: %s", tmp_stderr, tmp_stderr2,
+ strerror(errno));
+ failed();
+ }
+ fd_cpp_stderr = open(cpp_stderr, O_RDONLY | O_BINARY);
+ if (fd_cpp_stderr == -1) {
+ cc_log("Failed opening %s: %s", cpp_stderr, strerror(errno));
+ failed();
+ }
+ fd_real_stderr = open(tmp_stderr2, O_RDONLY | O_BINARY);
+ if (fd_real_stderr == -1) {
+ cc_log("Failed opening %s: %s", tmp_stderr2, strerror(errno));
+ failed();
+ }
+ fd_result = open(tmp_stderr, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
+ if (fd_result == -1) {
+ cc_log("Failed opening %s: %s", tmp_stderr, strerror(errno));
+ failed();
+ }
+ copy_fd(fd_cpp_stderr, fd_result);
+ copy_fd(fd_real_stderr, fd_result);
+ close(fd_cpp_stderr);
+ close(fd_real_stderr);
+ close(fd_result);
+ tmp_unlink(tmp_stderr2);
+ free(tmp_stderr2);
+ }
+
+ if (status != 0) {
+ int fd;
+ cc_log("Compiler gave exit status %d", status);
+ stats_update(STATS_STATUS);
+
+ fd = open(tmp_stderr, O_RDONLY | O_BINARY);
+ if (fd != -1) {
+ /* We can output stderr immediately instead of rerunning the compiler. */
+ copy_fd(fd, 2);
+ close(fd);
+ tmp_unlink(tmp_stderr);
+
+ x_exit(status);
+ }
+
+ tmp_unlink(tmp_stderr);
+ failed();
+ }
+
+ if (stat(output_obj, &st) != 0) {
+ cc_log("Compiler didn't produce an object file");
+ stats_update(STATS_NOOUTPUT);
+ failed();
+ }
+ if (st.st_size == 0) {
+ cc_log("Compiler produced an empty object file");
+ stats_update(STATS_EMPTYOUTPUT);
+ failed();
+ }
+
+ if (x_stat(tmp_stderr, &st) != 0) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ /* cache stderr */
+ if (!read_file(tmp_stderr, 0, &stderr_d, &stderr_l)) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ tmp_unlink(tmp_stderr);
+
+ if (output_dia) {
+ if (x_stat(output_dia, &st) != 0) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ /* cache dia */
+ if (!read_file(output_dia, 0, &dia_d, &dia_l)) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ }
+
+ /* cache output */
+ if (!read_file(output_obj, 0, &obj_d, &obj_l)) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+
+ if (generating_dependencies) {
+ if (!read_file(output_dep, 0, &dep_d, &dep_l)) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ }
+
+ if (memccached_set(cached_key, obj_d, stderr_d, dia_d, dep_d,
+ obj_l, stderr_l, dia_l, dep_l) < 0) {
+ stats_update(STATS_ERROR);
+ failed();
+ }
+
+ cc_log("Storing %s in memcached", cached_key);
+
+ stats_update(STATS_TOCACHE);
+
+ /* Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can
+ * be done almost anywhere, but we might as well do it near the end as we
+ * save the stat call if we exit early.
+ */
+ {
+ char *first_level_dir = dirname(stats_file);
+ if (create_cachedirtag(first_level_dir) != 0) {
+ cc_log("Failed to create %s/CACHEDIR.TAG (%s)\n",
+ first_level_dir, strerror(errno));
+ stats_update(STATS_ERROR);
+ failed();
+ }
+ free(first_level_dir);
+
+ /* Remove any CACHEDIR.TAG on the cache_dir level where it was located in
+ * previous ccache versions. */
+ if (getpid() % 1000 == 0) {
+ char *path = format("%s/CACHEDIR.TAG", conf->cache_dir);
+ x_unlink(path);
+ free(path);
+ }
+ }
+
+ /* Everything OK. */
+ send_cached_stderr();
+ update_manifest_file();
+
+ free(tmp_stderr);
+ free(tmp_stdout);
+}
+#endif
+
// Find the object file name by running the compiler in preprocessor mode.
// Returns the hash as a heap-allocated hex string.
static struct file_hash *
@@ -1408,6 +1719,7 @@ static void
update_cached_result_globals(struct file_hash *hash)
{
char *object_name = format_hash_as_string(hash->hash, hash->size);
+ cached_key = strdup(object_name);
cached_obj_hash = hash;
cached_obj = get_path_in_cache(object_name, ".o");
cached_stderr = get_path_in_cache(object_name, ".stderr");
@@ -1599,6 +1911,11 @@ calculate_common_hash(struct args *args, struct mdfour *hash)
static struct file_hash *
calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode)
{
+#if HAVE_LIBMEMCACHED
+ char *data;
+ size_t size;
+#endif
+
if (direct_mode) {
hash_delimiter(hash, "manifest version");
hash_int(hash, MANIFEST_VERSION);
@@ -1791,7 +2108,27 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode)
}
char *manifest_name = hash_result(hash);
manifest_path = get_path_in_cache(manifest_name, ".manifest");
- free(manifest_name);
+ /* Check if the manifest file is there. */
+ struct stat st;
+ if (stat(manifest_path, &st) != 0) {
+#if HAVE_LIBMEMCACHED
+ void *cache = NULL;
+#endif
+ cc_log("Manifest file %s not in cache", manifest_path);
+#if HAVE_LIBMEMCACHED
+ if (strlen(conf->memcached_conf) > 0) {
+ cc_log("Getting %s from memcached", manifest_name);
+ cache = memccached_raw_get(manifest_name, &data, &size);
+ }
+ if (cache) {
+ cc_log("Added object file hash to %s", manifest_path);
+ write_file(data, manifest_path, size);
+ stats_update_size(size, 1);
+ free(cache);
+ } else
+#endif
+ return NULL;
+ }
cc_log("Looking for object file hash in %s", manifest_path);
object_hash = manifest_get(conf, manifest_path);
if (object_hash) {
@@ -1828,8 +2165,13 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode)
// Try to return the compile result from cache. If we can return from cache
// then this function exits with the correct status code, otherwise it returns.
static void
-from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest)
+from_fscache(enum fromcache_call_mode mode, bool put_object_in_manifest)
{
+#if HAVE_LIBMEMCACHED
+ char *data_obj, *data_stderr, *data_dia, *data_dep;
+ size_t size_obj, size_stderr, size_dia, size_dep;
+#endif
+
// The user might be disabling cache hits.
if (conf->recache) {
return;
@@ -1837,7 +2179,33 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest)
struct stat st;
if (stat(cached_obj, &st) != 0) {
+#if HAVE_LIBMEMCACHED
+ void *cache = NULL;
+#endif
cc_log("Object file %s not in cache", cached_obj);
+#if HAVE_LIBMEMCACHED
+ if (strlen(conf->memcached_conf) > 0 &&
+ !using_split_dwarf &&
+ !generating_coverage) {
+ cc_log("Getting %s from memcached", cached_key);
+ cache = memccached_get(cached_key,
+ &data_obj, &data_stderr, &data_dia, &data_dep,
+ &size_obj, &size_stderr, &size_dia, &size_dep);
+ }
+ if (cache) {
+ put_data_in_cache(data_obj, size_obj, cached_obj);
+ if (size_stderr > 0) {
+ put_data_in_cache(data_stderr, size_stderr, cached_stderr);
+ }