Cassandra MAM backend tests for travis [not merge]

default.spec

@@ -8,38 +8,9 @@
 %% For more info see:
 %% http://www.erlang.org/doc/apps/common_test/run_test_chapter.html#test_specifications
 
-{suites, "tests", adhoc_SUITE}.
-{suites, "tests", amp_SUITE}.
-{suites, "tests", anonymous_SUITE}.
-{suites, "tests", bosh_SUITE}.
-{suites, "tests", carboncopy_SUITE}.
-{suites, "tests", cluster_commands_SUITE}.
-{suites, "tests", conf_reload_SUITE}.
-{suites, "tests", connect_SUITE}.
-{suites, "tests", ejabberdctl_SUITE}.
-{suites, "tests", last_SUITE}.
-{suites, "tests", login_SUITE}.
 {suites, "tests", mam_SUITE}.
-{suites, "tests", metrics_api_SUITE}.
-{suites, "tests", metrics_c2s_SUITE}.
-{suites, "tests", metrics_roster_SUITE}.
-{suites, "tests", metrics_register_SUITE}.
-{suites, "tests", metrics_session_SUITE}.
-{suites, "tests", muc_SUITE}.
-{suites, "tests", offline_SUITE}.
-{suites, "tests", presence_SUITE}.
-{suites, "tests", privacy_SUITE}.
-{suites, "tests", private_SUITE}.
-{suites, "tests", s2s_SUITE}.
-{suites, "tests", shared_roster_SUITE}.
-{suites, "tests", sic_SUITE}.
-{suites, "tests", sm_SUITE}.
-{suites, "tests", system_monitor_SUITE}.
-{suites, "tests", users_api_SUITE}.
-{suites, "tests", vcard_simple_SUITE}.
-{suites, "tests", websockets_SUITE}.
 {config, ["test.config"]}.
 {logdir, "ct_report"}.
-{ct_hooks, [ct_tty_hook, ct_mongoose_hook]}.
+{ct_hooks, [ct_tty_hook, ct_mongoose_hook, ct_travis_hook]}.
 %%To enable printing group and case enters on server side
 %%{ct_hooks, [{ct_tty_hook, [print_group, print_case]}]}.

full.spec

@@ -1,35 +1,2 @@
-{suites, "tests", anonymous_SUITE}.
-{suites, "tests", bosh_SUITE}.
-{suites, "tests", s2s_SUITE}.
-%%{suites, "tests", snmp_SUITE}.
-%%{suites, "tests", snmp_c2s_SUITE}.
-%%{suites, "tests", snmp_register_SUITE}.
-%%{suites, "tests", snmp_roster_SUITl}.
-%%{suites, "tests", snmp_session_SUITE}.
-%%{suites, "tests", snmp_table_SUITE}.
-
 %% taken from default.spec
-{suites, "tests", adhoc_SUITE}.
-{suites, "tests", conf_reload_SUITE}.
-{suites, "tests", ejabberdctl_SUITE}.
-{suites, "tests", last_SUITE}.
-{suites, "tests", login_SUITE}.
-{suites, "tests", muc_SUITE}.
 {suites, "tests", mam_SUITE}.
-{suites, "tests", offline_SUITE}.
-{suites, "tests", presence_SUITE}.
-{suites, "tests", privacy_SUITE}.
-{suites, "tests", private_SUITE}.
-{suites, "tests", sic_SUITE}.
-{suites, "tests", users_api_SUITE}.
-{suites, "tests", vcard_simple_SUITE}.
-{suites, "tests", websockets_SUITE}.
-{suites, "tests", metrics_api_SUITE}.
-{suites, "tests", metrics_c2s_SUITE}.
-{suites, "tests", metrics_roster_SUITE}.
-{suites, "tests", metrics_register_SUITE}.
-{suites, "tests", metrics_session_SUITE}.
-{suites, "tests", system_monitor_SUITE}.
-{suites, "tests", carboncopy_SUITE}.
-{config, ["test.config"]}.
-{logdir, "ct_report"}.

src/ct_travis_hook.erl

@@ -0,0 +1,155 @@
+%%% @doc Common Test Example Common Test Hook module.
+-module(ct_travis_hook).
+
+%% Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-record(state, { log_fd, test_n, group_test_n }).
+
+%% @doc Return a unique id for this CTH.
+id(Opts) ->
+    ct_travis_hook.
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. 
+init(Id, Opts) ->
+    ct:pal("Init ct_travis_hook", []),
+    {ok, FD} = file:open("/tmp/ct_travis_hook.log", [append, write]),
+    {ok, #state{log_fd=FD}}.
+
+%% @doc Called before init_per_suite is called. 
+pre_init_per_suite(Suite,Config,State) ->
+    State2 = reset_test_n(State),
+    {Config, fold_start(suite, Suite, State2)}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(Suite,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before end_per_suite. 
+pre_end_per_suite(Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after end_per_suite. 
+post_end_per_suite(Suite,Config,Return,State) ->
+    {Return, fold_end(suite, Suite, Return, State)}.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(Group,Config,State) ->
+    State2 = set_group_test_n(State),
+    State3 = fold_start(group, Group, State2),
+    {Config, State3}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(Group,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called after each end_per_group. 
+pre_end_per_group(Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each end_per_group. 
+post_end_per_group(Group,Config,Return,State) ->
+    {Return, fold_end(group, Group, Return, State)}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(TC,Config,State) ->
+    {Config, fold_start(test, TC, State)}.
+
+%% @doc Called after each test case.
+post_end_per_testcase(TC,Config,Return,State) ->
+    State2 = fold_end(test, TC, Return, State),
+    State3 = inc_test_n(State2),
+    {Return, State3}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(TC, Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing.  
+on_tc_skip(TC, Reason, State) ->
+    State.
+
+%% @doc Called when the scope of the CTH is done
+terminate(State) ->
+    ok.
+
+%% --------------------------------------------------------------------
+%% Private
+
+%% Same as:
+%% echo -en "travis_fold:start:Name\\r"
+%% echo "Name"
+fold_start(Type, Name, State=#state{log_fd=FD}) ->
+    STestN = get_and_print_n(Type, State),
+    SType = print_type(Type),
+    io:format(FD, "travis_fold:start:~s~p.~s\r~p~n",
+              [SType, Name, STestN, Name]),
+    State.
+
+%% Same as:
+%% echo -en "travis_fold:end:Name\\r"
+fold_end(Type, Name, Return, State=#state{log_fd=FD}) ->
+    STestN = get_and_print_n(Type, State),
+    SType = print_type(Type),
+    SReturn = print_return(Return),
+    io:format(FD, "travis_fold:end:~s~p.~s\r~s",
+              [SType, Name, STestN, SReturn]),
+    State.
+
+print_type(suite) -> "s.";
+print_type(group) -> "g.";
+print_type(test)  -> "t.".
+
+print_return({skip, Reason}) ->
+    io_lib:format("~nSkipped ~p~n", [Reason]);
+print_return({fail, Reason}) ->
+    io_lib:format("~nFailed ~p~n", [Reason]);
+print_return(_) ->
+    "".
+
+inc_test_n(State=#state{test_n=TestN}) ->
+    State#state{test_n=TestN+1}.
+
+reset_test_n(State=#state{test_n=TestN}) ->
+    State#state{test_n=1}.
+
+set_group_test_n(State=#state{test_n=TestN}) ->
+    State#state{group_test_n=TestN}.
+
+get_and_print_n(Type, State) ->
+    TestN = get_test_n(Type, State),
+    print_test_n(Type, TestN).
+
+get_test_n(test, State=#state{test_n=TestN}) ->
+    TestN;
+get_test_n(group, State=#state{group_test_n=TestN}) ->
+    TestN;
+get_test_n(suite, State=#state{}) ->
+    1.
+
+print_test_n(suite, _TestN) ->
+    ""; % do not print for suites, because TestN is always 1
+print_test_n(_Type, TestN) ->
+    integer_to_list(TestN).

test.config

@@ -201,7 +201,7 @@
 ]}.
 
 {mam, [
-    {skipped_configurations, [ca]}
+%   {skipped_configurations, [ca]}
 ]}.
 
 %% vim: ft=erlang

tests/mam_SUITE.erl

@@ -178,15 +178,38 @@ tests() ->
         not is_skipped(C, G)].
 
 groups() ->
-    [{full_group(C, G), Props, Tests}
+    [{full_group(C, G), Props, filter_tests(C, G, Tests)}
      || C <- configurations(), {G, Props, Tests} <- basic_groups(),
         not is_skipped(C, G)].
 
+filter_tests(C, G, Tests) ->
+    [Test || Test <- Tests, not is_test_skipped(C, G, Test)].
+
 is_skipped(odbc_mnesia_muc_cache, muc)         -> false;
 is_skipped(odbc_mnesia_muc_cache, muc_with_pm) -> false;
 is_skipped(odbc_mnesia_muc_cache, muc_rsm)     -> false;
+is_skipped(odbc_mnesia_muc_cache, _)           -> true;
+is_skipped(ca, rsm)                            ->
+    true; %% only with_rsm group supported, mod_mam_con_ca_arch missing_with_jid
 is_skipped(C, _) -> is_configuration_skipped(C).
 
+is_test_skipped(ca, muc_rsm,        muc_querying_for_all_messages_with_jid) ->
+    true; % mod_mam_muc_ca_arch     with_jid_not_supported
+is_test_skipped(ca, rsm,            pagination_offset5_opt_count) ->
+    true; % mod_mam_con_ca_arch     offset_not_supported
+is_test_skipped(ca, with_rsm,       pagination_offset5_opt_count) ->
+    true; % mod_mam_con_ca_arch     offset_not_supported
+is_test_skipped(ca, muc_rsm,        pagination_offset5_opt_count) ->
+    true; % mod_mam_muc_ca_arch     offset_not_supported
+is_test_skipped(ca, rsm,            pagination_offset5_opt_count_all) ->
+    true; % mod_mam_con_ca_arch     offset_not_supported
+is_test_skipped(ca, with_rsm,       pagination_offset5_opt_count_all) ->
+    true; % mod_mam_con_ca_arch     offset_not_supported
+is_test_skipped(ca, muc_rsm,        pagination_offset5_opt_count_all) ->
+    true; % mod_mam_muc_ca_arch     offset_not_supported
+is_test_skipped(_C, _G, _Test) ->
+    false.
+
 is_configuration_skipped(C) ->
     lists:member(C, skipped_configurations()).
 
@@ -246,9 +269,11 @@ muc_cases() ->
 muc_rsm_cases() ->
     rsm_cases().
 
+%% RSM cases with filtering by countact jid
 with_rsm_cases() ->
     rsm_cases().
 
+%% RSM cases without filtering by countact jid
 rsm_cases() ->
       [pagination_first5,
        pagination_last5,
@@ -292,15 +317,35 @@ init_per_group(Group, Config) ->
     B = basic_group(Group),
     ct:pal("Init per group ~p; configuration ~p; basic group ~p",
            [Group, C, B]),
-    Config1 = init_modules(C, B, Config),
-    init_state(C, B, Config1).
+    Config1 = try_init_modules(C, B, Config),
+    try_init_state(C, B, Config1).
 
 end_per_group(Group, Config) ->
     C = configuration(Group),
     B = basic_group(Group),
     Config1 = end_state(C, B, Config),
     end_modules(C, B, Config1).
 
+try_init_modules(C, B, Config) ->
+    try
+        init_modules(C, B, Config)
+    catch Class:Reason ->
+        Stacktrace = erlang:get_stacktrace(),
+        ct:pal("init_modules failed, configuration=~p, basic_group=~p, stacktrace=~p",
+               [C, B, Stacktrace]),
+        erlang:raise(Class, Reason, Stacktrace)
+    end.
+
+try_init_state(C, B, Config) ->
+    try
+        init_state(C, B, Config)
+    catch Class:Reason ->
+        Stacktrace = erlang:get_stacktrace(),
+        ct:pal("init_state failed, configuration=~p, basic_group=~p, stacktrace=~p",
+               [C, B, Stacktrace]),
+        erlang:raise(Class, Reason, Stacktrace)
+    end.
+
 init_modules(C, muc_rsm, Config) ->
     init_modules(C, muc, Config);