diff --git a/default.spec b/default.spec index 43ee010..087e8cd 100644 --- a/default.spec +++ b/default.spec @@ -8,38 +8,9 @@ %% For more info see: %% http://www.erlang.org/doc/apps/common_test/run_test_chapter.html#test_specifications -{suites, "tests", adhoc_SUITE}. -{suites, "tests", amp_SUITE}. -{suites, "tests", anonymous_SUITE}. -{suites, "tests", bosh_SUITE}. -{suites, "tests", carboncopy_SUITE}. -{suites, "tests", cluster_commands_SUITE}. -{suites, "tests", conf_reload_SUITE}. -{suites, "tests", connect_SUITE}. -{suites, "tests", ejabberdctl_SUITE}. -{suites, "tests", last_SUITE}. -{suites, "tests", login_SUITE}. {suites, "tests", mam_SUITE}. -{suites, "tests", metrics_api_SUITE}. -{suites, "tests", metrics_c2s_SUITE}. -{suites, "tests", metrics_roster_SUITE}. -{suites, "tests", metrics_register_SUITE}. -{suites, "tests", metrics_session_SUITE}. -{suites, "tests", muc_SUITE}. -{suites, "tests", offline_SUITE}. -{suites, "tests", presence_SUITE}. -{suites, "tests", privacy_SUITE}. -{suites, "tests", private_SUITE}. -{suites, "tests", s2s_SUITE}. -{suites, "tests", shared_roster_SUITE}. -{suites, "tests", sic_SUITE}. -{suites, "tests", sm_SUITE}. -{suites, "tests", system_monitor_SUITE}. -{suites, "tests", users_api_SUITE}. -{suites, "tests", vcard_simple_SUITE}. -{suites, "tests", websockets_SUITE}. {config, ["test.config"]}. {logdir, "ct_report"}. -{ct_hooks, [ct_tty_hook, ct_mongoose_hook]}. +{ct_hooks, [ct_tty_hook, ct_mongoose_hook, ct_travis_hook]}. %%To enable printing group and case enters on server side %%{ct_hooks, [{ct_tty_hook, [print_group, print_case]}]}. diff --git a/full.spec b/full.spec index 0384f09..660a3e6 100644 --- a/full.spec +++ b/full.spec @@ -1,35 +1,2 @@ -{suites, "tests", anonymous_SUITE}. -{suites, "tests", bosh_SUITE}. -{suites, "tests", s2s_SUITE}. -%%{suites, "tests", snmp_SUITE}. -%%{suites, "tests", snmp_c2s_SUITE}. -%%{suites, "tests", snmp_register_SUITE}. -%%{suites, "tests", snmp_roster_SUITl}. -%%{suites, "tests", snmp_session_SUITE}. -%%{suites, "tests", snmp_table_SUITE}. - %% taken from default.spec -{suites, "tests", adhoc_SUITE}. -{suites, "tests", conf_reload_SUITE}. -{suites, "tests", ejabberdctl_SUITE}. -{suites, "tests", last_SUITE}. -{suites, "tests", login_SUITE}. -{suites, "tests", muc_SUITE}. {suites, "tests", mam_SUITE}. -{suites, "tests", offline_SUITE}. -{suites, "tests", presence_SUITE}. -{suites, "tests", privacy_SUITE}. -{suites, "tests", private_SUITE}. -{suites, "tests", sic_SUITE}. -{suites, "tests", users_api_SUITE}. -{suites, "tests", vcard_simple_SUITE}. -{suites, "tests", websockets_SUITE}. -{suites, "tests", metrics_api_SUITE}. -{suites, "tests", metrics_c2s_SUITE}. -{suites, "tests", metrics_roster_SUITE}. -{suites, "tests", metrics_register_SUITE}. -{suites, "tests", metrics_session_SUITE}. -{suites, "tests", system_monitor_SUITE}. -{suites, "tests", carboncopy_SUITE}. -{config, ["test.config"]}. -{logdir, "ct_report"}. diff --git a/src/ct_travis_hook.erl b/src/ct_travis_hook.erl new file mode 100644 index 0000000..79e6f98 --- /dev/null +++ b/src/ct_travis_hook.erl @@ -0,0 +1,155 @@ +%%% @doc Common Test Example Common Test Hook module. +-module(ct_travis_hook). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-record(state, { log_fd, test_n, group_test_n }). + +%% @doc Return a unique id for this CTH. +id(Opts) -> + ct_travis_hook. + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, Opts) -> + ct:pal("Init ct_travis_hook", []), + {ok, FD} = file:open("/tmp/ct_travis_hook.log", [append, write]), + {ok, #state{log_fd=FD}}. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + State2 = reset_test_n(State), + {Config, fold_start(suite, Suite, State2)}. + +%% @doc Called after init_per_suite. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(Suite,Config,Return,State) -> + {Return, fold_end(suite, Suite, Return, State)}. + +%% @doc Called before each init_per_group. +pre_init_per_group(Group,Config,State) -> + State2 = set_group_test_n(State), + State3 = fold_start(group, Group, State2), + {Config, State3}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(Group,Config,Return,State) -> + {Return, fold_end(group, Group, Return, State)}. + +%% @doc Called before each test case. +pre_init_per_testcase(TC,Config,State) -> + {Config, fold_start(test, TC, State)}. + +%% @doc Called after each test case. +post_end_per_testcase(TC,Config,Return,State) -> + State2 = fold_end(test, TC, Return, State), + State3 = inc_test_n(State2), + {Return, State3}. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(State) -> + ok. + +%% -------------------------------------------------------------------- +%% Private + +%% Same as: +%% echo -en "travis_fold:start:Name\\r" +%% echo "Name" +fold_start(Type, Name, State=#state{log_fd=FD}) -> + STestN = get_and_print_n(Type, State), + SType = print_type(Type), + io:format(FD, "travis_fold:start:~s~p.~s\r~p~n", + [SType, Name, STestN, Name]), + State. + +%% Same as: +%% echo -en "travis_fold:end:Name\\r" +fold_end(Type, Name, Return, State=#state{log_fd=FD}) -> + STestN = get_and_print_n(Type, State), + SType = print_type(Type), + SReturn = print_return(Return), + io:format(FD, "travis_fold:end:~s~p.~s\r~s", + [SType, Name, STestN, SReturn]), + State. + +print_type(suite) -> "s."; +print_type(group) -> "g."; +print_type(test) -> "t.". + +print_return({skip, Reason}) -> + io_lib:format("~nSkipped ~p~n", [Reason]); +print_return({fail, Reason}) -> + io_lib:format("~nFailed ~p~n", [Reason]); +print_return(_) -> + "". + +inc_test_n(State=#state{test_n=TestN}) -> + State#state{test_n=TestN+1}. + +reset_test_n(State=#state{test_n=TestN}) -> + State#state{test_n=1}. + +set_group_test_n(State=#state{test_n=TestN}) -> + State#state{group_test_n=TestN}. + +get_and_print_n(Type, State) -> + TestN = get_test_n(Type, State), + print_test_n(Type, TestN). + +get_test_n(test, State=#state{test_n=TestN}) -> + TestN; +get_test_n(group, State=#state{group_test_n=TestN}) -> + TestN; +get_test_n(suite, State=#state{}) -> + 1. + +print_test_n(suite, _TestN) -> + ""; % do not print for suites, because TestN is always 1 +print_test_n(_Type, TestN) -> + integer_to_list(TestN). diff --git a/test.config b/test.config index a6f9b52..6d04877 100644 --- a/test.config +++ b/test.config @@ -201,7 +201,7 @@ ]}. {mam, [ - {skipped_configurations, [ca]} +% {skipped_configurations, [ca]} ]}. %% vim: ft=erlang diff --git a/tests/mam_SUITE.erl b/tests/mam_SUITE.erl index 168a79a..b9e7a61 100644 --- a/tests/mam_SUITE.erl +++ b/tests/mam_SUITE.erl @@ -178,15 +178,38 @@ tests() -> not is_skipped(C, G)]. groups() -> - [{full_group(C, G), Props, Tests} + [{full_group(C, G), Props, filter_tests(C, G, Tests)} || C <- configurations(), {G, Props, Tests} <- basic_groups(), not is_skipped(C, G)]. +filter_tests(C, G, Tests) -> + [Test || Test <- Tests, not is_test_skipped(C, G, Test)]. + is_skipped(odbc_mnesia_muc_cache, muc) -> false; is_skipped(odbc_mnesia_muc_cache, muc_with_pm) -> false; is_skipped(odbc_mnesia_muc_cache, muc_rsm) -> false; +is_skipped(odbc_mnesia_muc_cache, _) -> true; +is_skipped(ca, rsm) -> + true; %% only with_rsm group supported, mod_mam_con_ca_arch missing_with_jid is_skipped(C, _) -> is_configuration_skipped(C). +is_test_skipped(ca, muc_rsm, muc_querying_for_all_messages_with_jid) -> + true; % mod_mam_muc_ca_arch with_jid_not_supported +is_test_skipped(ca, rsm, pagination_offset5_opt_count) -> + true; % mod_mam_con_ca_arch offset_not_supported +is_test_skipped(ca, with_rsm, pagination_offset5_opt_count) -> + true; % mod_mam_con_ca_arch offset_not_supported +is_test_skipped(ca, muc_rsm, pagination_offset5_opt_count) -> + true; % mod_mam_muc_ca_arch offset_not_supported +is_test_skipped(ca, rsm, pagination_offset5_opt_count_all) -> + true; % mod_mam_con_ca_arch offset_not_supported +is_test_skipped(ca, with_rsm, pagination_offset5_opt_count_all) -> + true; % mod_mam_con_ca_arch offset_not_supported +is_test_skipped(ca, muc_rsm, pagination_offset5_opt_count_all) -> + true; % mod_mam_muc_ca_arch offset_not_supported +is_test_skipped(_C, _G, _Test) -> + false. + is_configuration_skipped(C) -> lists:member(C, skipped_configurations()). @@ -246,9 +269,11 @@ muc_cases() -> muc_rsm_cases() -> rsm_cases(). +%% RSM cases with filtering by countact jid with_rsm_cases() -> rsm_cases(). +%% RSM cases without filtering by countact jid rsm_cases() -> [pagination_first5, pagination_last5, @@ -292,8 +317,8 @@ init_per_group(Group, Config) -> B = basic_group(Group), ct:pal("Init per group ~p; configuration ~p; basic group ~p", [Group, C, B]), - Config1 = init_modules(C, B, Config), - init_state(C, B, Config1). + Config1 = try_init_modules(C, B, Config), + try_init_state(C, B, Config1). end_per_group(Group, Config) -> C = configuration(Group), @@ -301,6 +326,26 @@ end_per_group(Group, Config) -> Config1 = end_state(C, B, Config), end_modules(C, B, Config1). +try_init_modules(C, B, Config) -> + try + init_modules(C, B, Config) + catch Class:Reason -> + Stacktrace = erlang:get_stacktrace(), + ct:pal("init_modules failed, configuration=~p, basic_group=~p, stacktrace=~p", + [C, B, Stacktrace]), + erlang:raise(Class, Reason, Stacktrace) + end. + +try_init_state(C, B, Config) -> + try + init_state(C, B, Config) + catch Class:Reason -> + Stacktrace = erlang:get_stacktrace(), + ct:pal("init_state failed, configuration=~p, basic_group=~p, stacktrace=~p", + [C, B, Stacktrace]), + erlang:raise(Class, Reason, Stacktrace) + end. + init_modules(C, muc_rsm, Config) -> init_modules(C, muc, Config);