Improve parallel test runs

The main improvement is to separate the per-app makefile targets early, during
templating phase, so they never share a common app structure during runtime.
That lets us avoid reconfiguring the data directory at runtime, which
effectively is an almost full restart internally since we're tearing down all
the couch servers. So we're saving an extra app shutdown + restart cycle per
app with the new scheme and improving the isolation.

The new scheme uses the `tmp/$app` namespace for each app. After the tests run
any unsuccessful `tmp/$app` directories will be left behind so they can be
inspected. This is especially useful for looking at the failed couch.log
entries. There is a separate catlogs target to concatenate failed app couch.log
files though maybe keeping them separately may be even easier to inspect. Since
if app a failed, we'd just rather look at app a's logs and not app c's logs.

Since setup_eunit rebar template now takes an app argument, opted to remove
some older eunit targets keeping only the new ones supporting parallel
execution. Otherwise, would have to add a special case statement in the plugin
for no-app vs one app cases.

Previously, `make eunit apps=a,b,c` did not run apps `a`, `b` and `c` as
separate goals in parallel execution. They were passed in as the `a,b,c` goal.
To fix that parse the apps into individual space separated goals, so they can
be sped up by parallel execution as well.

To keep the apps as isolated from each other as possible make
build_eunit_config also create separate etc folders, use separate BUILDDIR
macros and each apps only uses its own fixtures. Also use 0-valued ports the
eunit template to avoid any chance of port collision and resulting flakiness.

Rename SUBDIRS to EUNIT_SUBDIRS since they used for eunit tests currently only.
Also, exclude docs and fauxton apps from the target list, since they are not
proper erlang apps and don't have eunit tests.

Newer Erlangs have `file:del_dir_r/1` so use that in cleanup_dirs to save a few
lines of code.

Author: nickva

Makefile

@@ -168,53 +168,27 @@ check: all
 	@$(MAKE) nouveau-test
 
 ifdef apps
-SUBDIRS = $(apps)
+EUNIT_SUBDIRS = $(strip $(subst $(comma),$(space),$(apps)))
 else
-SUBDIRS=$(shell ls src)
+EUNIT_SUBDIRS = $(filter-out fauxton docs, $(shell ls src))
 endif
 
-# Used for comparing behaviour against he new `eunit` target, delete once we
-# are happy with the new `eunit`.
-.PHONY: old-eunit
-old-eunit: export BUILDDIR = $(CURDIR)
-old-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config
-old-eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js
-old-eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
-old-eunit:
-	@COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) setup_eunit 2> /dev/null
-	@for dir in $(SUBDIRS); do \
-     COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$$dir || exit 1; \
-  done
-
 # target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options
-.PHONY: eunit $(SUBDIRS)
-eunit: export BUILDDIR = $(CURDIR)
+.PHONY: eunit $(EUNIT_SUBDIRS)
 eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config
 eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js
 eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
-eunit: ${SUBDIRS}
-	@cat tmp/couchdb-tests/*/log/couch.log > tmp/couch.log
-	@rm -rf tmp/couchdb-tests/*
-
-$(SUBDIRS): setup-eunit
-	@COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ #|| exit 1
-
-setup-eunit: export BUILDDIR = $(CURDIR)
-setup-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config
-setup-eunit:
-	@$(REBAR) setup_eunit 2> /dev/null
-
-just-eunit: export BUILDDIR = $(CURDIR)
-just-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config
-just-eunit:
-	@$(REBAR) -r eunit $(EUNIT_OPTS)
-
-.PHONY: soak-eunit
-soak-eunit: export BUILDDIR = $(CURDIR)
-soak-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config
-soak-eunit: couch-core
-	@$(REBAR) setup_eunit 2> /dev/null
-	while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done
+eunit: ${EUNIT_SUBDIRS}
+
+$(EUNIT_SUBDIRS):
+	@rm -rf tmp/$@
+	@$(REBAR) setup_eunit app=$@ >/dev/null
+	@BUILDDIR=$(CURDIR)/tmp/$@ COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ && rm -rf tmp/$@
+
+# cat together couch_log files after running eunit test
+.PHONY: catlogs
+catlogs:
+	@ls tmp/*/couch.log 2>/dev/null | xargs cat > tmp/couch.log || true
 
 # target: erlfmt-check - Check Erlang source code formatting
 erlfmt-check:
@@ -316,6 +290,7 @@ build-report:
 	build-aux/show-test-results.py --suites=10 --tests=10 > test-results.log || true
 	cat ./dev/logs/node1.log || true
 	cat ./dev/logs/nouveau.log || true
+	$(MAKE) catlogs || true
 	cat ./tmp/couch.log || true
 	cat test-results.log || true
 

rel/plugins/eunit_plugin.erl

@@ -25,37 +25,56 @@ is_base_dir(RebarConf) ->
     filename:absname(rebar_utils:get_cwd()) =:=
         rebar_config:get_xconf(RebarConf, base_dir, undefined).
 
-build_eunit_config(Config0, AppFile) ->
+build_eunit_config(Config, AppFile) ->
     Cwd = filename:absname(rebar_utils:get_cwd()),
-    DataDir = Cwd ++ "/tmp/data",
-    ViewIndexDir = Cwd ++ "/tmp/data",
-    StateDir = Cwd ++ "/tmp/data",
-    TmpDataDir = Cwd ++ "/tmp/tmp_data",
-    LogDir = Cwd ++ "/tmp",
-    cleanup_dirs([DataDir, TmpDataDir]),
-    Config1 = rebar_config:set_global(Config0, template, "setup_eunit"),
-    Config2 = rebar_config:set_global(Config1, prefix, Cwd),
+    App = rebar_config:get_global(Config, app, undefined),
+    case is_list(App) of
+        true -> ok;
+        false -> error(app_parameter_must_be_defined)
+    end,
+    case re:run(App, "^[_a-z0-9]+$") of
+        nomatch ->
+            error({app_parameter_must_be_just_one_app, App});
+        {match, _} ->
+            Prefix = Cwd ++ "/tmp/" ++ App,
+            DataDir = Prefix ++ "/data",
+            ViewIndexDir = Prefix ++ "/data",
+            StateDir = Prefix ++ "/data",
+            TmpDataDir = Prefix ++ "/tmp_data",
+            EtcDir = Prefix ++ "/etc",
+            LogDir = Prefix,
+            build_config(
+                Config,
+                AppFile,
+                Prefix,
+                DataDir,
+                ViewIndexDir,
+                StateDir,
+                TmpDataDir,
+                EtcDir,
+                LogDir
+            )
+    end.
+
+build_config(Config, AppFile, Prefix, DataDir, ViewIndexDir, StateDir, TmpDataDir, EtcDir, LogDir) ->
+    cleanup_dirs([DataDir, TmpDataDir, EtcDir]),
+    Config1 = rebar_config:set_global(Config, template, "setup_eunit"),
+    Config2 = rebar_config:set_global(Config1, prefix, Prefix),
     Config3 = rebar_config:set_global(Config2, data_dir, DataDir),
     Config4 = rebar_config:set_global(Config3, view_index_dir, ViewIndexDir),
     Config5 = rebar_config:set_global(Config4, log_dir, LogDir),
-    Config = rebar_config:set_global(Config5, state_dir, StateDir),
-    rebar_templater:create(Config, AppFile).
+    Config6 = rebar_config:set_global(Config5, etc_dir, EtcDir),
+    Config7 = rebar_config:set_global(Config6, tmp_data, TmpDataDir),
+    Config8 = rebar_config:set_global(Config7, state_dir, StateDir),
+    rebar_templater:create(Config8, AppFile).
 
 cleanup_dirs(Dirs) ->
     lists:foreach(
         fun(Dir) ->
             case filelib:is_dir(Dir) of
-                true -> del_dir(Dir);
+                true -> file:del_dir_r(Dir);
                 false -> ok
             end
         end,
         Dirs
     ).
-
-del_dir(Dir) ->
-    All = filelib:wildcard(Dir ++ "/**"),
-    {Dirs, Files} = lists:partition(fun filelib:is_dir/1, All),
-    ok = lists:foreach(fun file:delete/1, Files),
-    SortedDirs = lists:sort(fun(A, B) -> length(A) > length(B) end, Dirs),
-    ok = lists:foreach(fun file:del_dir/1, SortedDirs),
-    ok = file:del_dir(Dir).

setup_eunit.template

@@ -1,21 +1,24 @@
 {variables, [
     {package_author_name, "The Apache Software Foundation"},
-    {cluster_port, 5984},
-    {backend_port, 5986},
-    {prometheus_port, 17986},
+    {cluster_port, 0},
+    {backend_port, 0},
+    {prometheus_port, 0},
     {node_name, "-name couchdbtest@127.0.0.1"},
 
     {data_dir, "/tmp"},
     {prefix, "/tmp"},
     {view_index_dir, "/tmp"},
     {state_dir, "/tmp"},
-    {log_dir, "/tmp"}
+    {log_dir, "/tmp"},
+    {etc_dir, "/tmp"},
+    {tmp_data, "/tmp"}
 ]}.
-{dir, "tmp"}.
-{dir, "tmp/etc"}.
-{dir, "tmp/data"}.
-{dir, "tmp/tmp_data"}.
-{template, "rel/overlay/etc/default.ini", "tmp/etc/default_eunit.ini"}.
-{template, "rel/overlay/etc/local.ini", "tmp/etc/local_eunit.ini"}.
-{template, "rel/files/eunit.ini", "tmp/etc/eunit.ini"}.
-{template, "rel/overlay/etc/vm.args", "tmp/etc/vm.args"}.
+{dir, "{{prefix}}"}.
+{dir, "{{etc_dir}}"}.
+{dir, "{{data_dir}}"}.
+{dir, "{{tmp_data}}"}.
+
+{template, "rel/overlay/etc/default.ini", "{{prefix}}/etc/default_eunit.ini"}.
+{template, "rel/overlay/etc/local.ini", "{{prefix}}/etc/local_eunit.ini"}.
+{template, "rel/files/eunit.ini", "{{prefix}}/etc/eunit.ini"}.
+{template, "rel/overlay/etc/vm.args", "{{prefix}}/etc/vm.args"}.

src/config/test/config_tests.erl

@@ -24,7 +24,7 @@
 -define(RESTART_TIMEOUT_IN_MILLISEC, 3000).
 
 -define(CONFIG_FIXTURESDIR,
-    filename:join([?BUILDDIR(), "src", "config", "test", "fixtures"])
+    ?ABS_PATH("test/fixtures")
 ).
 
 -define(CONFIG_DEFAULT_TEST,

src/couch/include/couch_eunit.hrl

@@ -25,22 +25,23 @@
         end
     end).
 -define(CONFIG_DEFAULT,
-    filename:join([?BUILDDIR(), "tmp", "etc", "default_eunit.ini"])).
+    filename:join([?BUILDDIR(), "etc", "default_eunit.ini"])).
 -define(CONFIG_CHAIN, [
     ?CONFIG_DEFAULT,
-    filename:join([?BUILDDIR(), "tmp", "etc", "local_eunit.ini"]),
-    filename:join([?BUILDDIR(), "tmp", "etc", "eunit.ini"])]).
--define(FIXTURESDIR,
-    filename:join([?BUILDDIR(), "src", "couch", "test", "eunit", "fixtures"])).
+    filename:join([?BUILDDIR(), "etc", "local_eunit.ini"]),
+    filename:join([?BUILDDIR(), "etc", "eunit.ini"])]).
 -define(TEMPDIR,
-    filename:join([?BUILDDIR(), "tmp", "tmp_data"])).
+    filename:join([?BUILDDIR(), "tmp_data"])).
 
 -define(APPDIR, filename:dirname(element(2, file:get_cwd()))).
 %% Account for the fact that source files are in src/<app>/.eunit/<module>.erl
 %% when run from eunit
 -define(ABS_PATH(File), %% src/<app>/.eunit/<module>.erl
     filename:join([?APPDIR, File])).
 
+-define(FIXTURESDIR,
+    ?ABS_PATH("test/eunit/fixtures")).
+
 -define(tempfile,
     fun() ->
         Suffix = couch_uuids:random(),

src/couch/src/test_util.erl

@@ -44,7 +44,7 @@
 -include("couch_db_int.hrl").
 -include("couch_bt_engine.hrl").
 
--record(test_context, {mocked = [], started = [], module, dir}).
+-record(test_context, {mocked = [], started = [], module}).
 
 -define(DEFAULT_APPS, [inets, ibrowse, ssl, config, couch_epi, couch_event, couch]).
 
@@ -81,61 +81,21 @@ start_couch() ->
 start_couch(ExtraApps) ->
     start_couch(?CONFIG_CHAIN, ExtraApps).
 
-% This function starts CouchDB with optional extra apps in a dedicated
-% directory under tmp/couchdb-tests/<uuid>/ — That is, all instances
-% run isolated so that if more than one runs at a time, it does not
-% interfere with any of the others.
-% The sub-directories here are etc/ log/ and data/ for configuration
-% logs and databases and view indexes respectively.
-% The function copies the initial bespoke test-ini files to the new
-% random directory and configures CouchDB to use those.
-% The two pieces of state created here are the random dir in the file system and
-% the process env var {config, ini_files} that is read by couch_config on
-% startup. These need to be reset at the right points in time.
-% the random dir is deleted and the env var reset when CouchDB is stopped.
-% Note: there is currently a case where stop_couch/0 could be called and
-% the file system cleanup can’t be run because we don’t get passed the test
-% context with the random directory value. At the moment stop_couch/0 is not
-% used anywhere, tho. Maybe we should remove it?
 start_couch(IniFiles, ExtraApps) ->
-    RandomDir = filename:join([builddir(), "tmp", "couchdb-tests", couch_uuids:random()]),
-    RandomEtcDir = filename:join([RandomDir, "etc"]),
-    RandomDataDir = ?b2l(filename:join([RandomDir, "data"])),
-    RandomLogFile = ?b2l(filename:join([RandomDir, "log", "couch.log"])),
-
-    ok = filelib:ensure_path(RandomDir),
-    ok = filelib:ensure_path(RandomEtcDir),
-    ok = filelib:ensure_path(RandomDataDir),
-    ok = filelib:ensure_dir(RandomLogFile),
-
-    RandomIniFiles = lists:map(
-        fun(SourceFile) ->
-            TargetFileName = lists:last(filename:split(SourceFile)),
-            TargetFile = filename:join([RandomEtcDir, TargetFileName]),
-            {ok, _} = file:copy(SourceFile, TargetFile),
-            ?b2l(TargetFile)
-        end,
-        IniFiles
-    ),
     load_applications_with_stats(),
-    ok = application:set_env(config, ini_files, RandomIniFiles),
+    ok = application:set_env(config, ini_files, IniFiles),
     Apps = start_applications(?DEFAULT_APPS ++ ExtraApps),
-
-    ok = config:set("couchdb", "database_dir", RandomDataDir, false),
-    ok = config:set("couchdb", "view_index_dir", RandomDataDir, false),
-    ok = config:set("log", "file", RandomLogFile, false),
     ok = config:delete("compactions", "_default", false),
-    #test_context{started = Apps, dir = RandomDir}.
+    #test_context{started = Apps}.
 
 stop_couch() ->
     ok = stop_applications(?DEFAULT_APPS).
 
-stop_couch(#test_context{started = Apps, dir = RandomDir}) ->
-    % deletion now happens in Makefile
-    % file:del_dir_r(RandomDir),
+stop_couch(#test_context{started = Apps}) ->
     stop_applications(Apps);
 stop_couch(_) ->
     stop_couch().
+
 with_couch_server_restart(Fun) ->
     Servers = couch_server:names(),
     test_util:with_processes_restart(Servers, Fun).
@@ -415,7 +375,7 @@ mock(couch_stats) ->
     ok.
 
 load_applications_with_stats() ->
-    Wildcard = filename:join([?BUILDDIR(), "src/*/priv/stats_descriptions.cfg"]),
+    Wildcard = ?ABS_PATH("../*/priv/stats_descriptions.cfg"),
     [application:load(stats_file_to_app(File)) || File <- filelib:wildcard(Wildcard)],
     ok.
 

src/couch_replicator/test/eunit/fixtures/logo.png

@@ -106,7 +106,7 @@ teardown(Ctx) ->
     catch application:stop(mem3),
     config:delete("cluster", "seedlist", false),
     Filename = config:get("mem3", "nodes_db", "_nodes") ++ ".couch",
-    file:delete(filename:join([?BUILDDIR(), "tmp", "data", Filename])),
+    file:delete(filename:join([?BUILDDIR(), "data", Filename])),
     case config:get("couch_httpd_auth", "authentication_db") of
         undefined -> ok;
         DbName -> couch_server:delete(list_to_binary(DbName), [])

src/mem3/test/eunit/mem3_seeds_test.erl

@@ -68,7 +68,7 @@ teardown(Ctx) ->
     catch application:stop(mem3),
     os:unsetenv("COUCHDB_ZONE"),
     Filename = config:get("mem3", "nodes_db", "_nodes") ++ ".couch",
-    file:delete(filename:join([?BUILDDIR(), "tmp", "data", Filename])),
+    file:delete(filename:join([?BUILDDIR(), "data", Filename])),
     case config:get("couch_httpd_auth", "authentication_db") of
         undefined -> ok;
         DbName -> couch_server:delete(list_to_binary(DbName), [])