Merge branch 'TT_RETRIABLE'

* TT_RETRIABLE:
  Mark a lot of flacky tests with TT_RETRIABLE (for linux/win32 only)
  regress: introduce TT_RETRIABLE

Fixes: #704

test/regress.c

@@ -3416,14 +3416,14 @@ struct testcase_t main_testcases[] = {
 
 	BASIC(bad_assign, TT_FORK|TT_NEED_BASE|TT_NO_LOGS),
 	BASIC(bad_reentrant, TT_FORK|TT_NEED_BASE|TT_NO_LOGS),
-	BASIC(active_later, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR),
+	BASIC(active_later, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR|TT_RETRIABLE),
 	BASIC(event_remove_timeout, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR),
 
 	/* These are still using the old API */
 	LEGACY(persistent_timeout, TT_FORK|TT_NEED_BASE),
 	{ "persistent_timeout_jump", test_persistent_timeout_jump, TT_FORK|TT_NEED_BASE, &basic_setup, NULL },
 	{ "persistent_active_timeout", test_persistent_active_timeout,
-	  TT_FORK|TT_NEED_BASE, &basic_setup, NULL },
+	  TT_FORK|TT_NEED_BASE|TT_RETRIABLE, &basic_setup, NULL },
 	LEGACY(priorities, TT_FORK|TT_NEED_BASE),
 	BASIC(priority_active_inversion, TT_FORK|TT_NEED_BASE),
 	{ "common_timeout", test_common_timeout, TT_FORK|TT_NEED_BASE,
@@ -3476,7 +3476,7 @@ struct testcase_t main_testcases[] = {
 #endif
 #ifdef EVENT__HAVE_PTHREADS
 	/** TODO: support win32 */
-	LEGACY(del_wait, TT_ISOLATED|TT_NEED_THREADS),
+	LEGACY(del_wait, TT_ISOLATED|TT_NEED_THREADS|TT_RETRIABLE),
 	LEGACY(del_notify, TT_ISOLATED|TT_NEED_THREADS),
 #endif
 

test/regress_http.c

@@ -4826,6 +4826,8 @@ http_newreqcb_test(void *arg)
 #define HTTP_CAST_ARG(a) ((void *)(a))
 #define HTTP_OFF_N(title, name, arg) \
 	{ #title, http_##name##_test, TT_ISOLATED|TT_OFF_BY_DEFAULT, &basic_setup, HTTP_CAST_ARG(arg) }
+#define HTTP_RET_N(title, name, arg) \
+	{ #title, http_##name##_test, TT_ISOLATED|TT_RETRIABLE, &basic_setup, HTTP_CAST_ARG(arg) }
 #define HTTP_N(title, name, arg) \
 	{ #title, http_##name##_test, TT_ISOLATED, &basic_setup, HTTP_CAST_ARG(arg) }
 #define HTTP(name) HTTP_N(name, name, NULL)
@@ -4881,17 +4883,17 @@ struct testcase_t http_testcases[] = {
 	HTTP(simple_nonconformant),
 
 	HTTP_N(cancel, cancel, BASIC),
-	HTTP_N(cancel_by_host, cancel, BY_HOST),
+	HTTP_RET_N(cancel_by_host, cancel, BY_HOST),
+	HTTP_RET_N(cancel_by_host_inactive_server, cancel, BY_HOST | INACTIVE_SERVER),
 	HTTP_N(cancel_by_host_no_ns, cancel, BY_HOST | NO_NS),
-	HTTP_N(cancel_by_host_inactive_server, cancel, BY_HOST | INACTIVE_SERVER),
 	HTTP_N(cancel_inactive_server, cancel, INACTIVE_SERVER),
 	HTTP_N(cancel_by_host_no_ns_inactive_server, cancel, BY_HOST | NO_NS | INACTIVE_SERVER),
 	HTTP_OFF_N(cancel_by_host_server_timeout, cancel, BY_HOST | INACTIVE_SERVER | SERVER_TIMEOUT),
 	HTTP_OFF_N(cancel_server_timeout, cancel, INACTIVE_SERVER | SERVER_TIMEOUT),
 	HTTP_OFF_N(cancel_by_host_no_ns_server_timeout, cancel, BY_HOST | NO_NS | INACTIVE_SERVER | SERVER_TIMEOUT),
 	HTTP_OFF_N(cancel_by_host_ns_timeout_server_timeout, cancel, BY_HOST | NO_NS | NS_TIMEOUT | INACTIVE_SERVER | SERVER_TIMEOUT),
-	HTTP_N(cancel_by_host_ns_timeout, cancel, BY_HOST | NO_NS | NS_TIMEOUT),
+	HTTP_RET_N(cancel_by_host_ns_timeout, cancel, BY_HOST | NO_NS | NS_TIMEOUT),
-	HTTP_N(cancel_by_host_ns_timeout_inactive_server, cancel, BY_HOST | NO_NS | NS_TIMEOUT | INACTIVE_SERVER),
+	HTTP_RET_N(cancel_by_host_ns_timeout_inactive_server, cancel, BY_HOST | NO_NS | NS_TIMEOUT | INACTIVE_SERVER),
 
 	HTTP(virtual_host),
 	HTTP(post),

test/regress_thread.c

@@ -564,8 +564,8 @@ end:
 	;
 }
 
-#define TEST(name)							\
+#define TEST(name, f)							\
-	{ #name, thread_##name, TT_FORK|TT_NEED_THREADS|TT_NEED_BASE,	\
+	{ #name, thread_##name, TT_FORK|TT_NEED_THREADS|TT_NEED_BASE|(f),	\
 	  &basic_setup, NULL }
 
 struct testcase_t thread_testcases[] = {
@@ -575,7 +575,7 @@ struct testcase_t thread_testcases[] = {
 	{ "forking", thread_basic, TT_FORK|TT_NEED_THREADS|TT_NEED_BASE,
 	  &basic_setup, (char*)"forking" },
 #endif
-	TEST(conditions_simple),
+	TEST(conditions_simple, TT_RETRIABLE),
 	{ "deferred_cb_skew", thread_deferred_cb_skew,
 	  TT_FORK|TT_NEED_THREADS|TT_OFF_BY_DEFAULT,
 	  &basic_setup, NULL },
@@ -583,7 +583,7 @@ struct testcase_t thread_testcases[] = {
 	/****** XXX TODO FIXME windows seems to be having some timing trouble,
 	 * looking into it now. / ellzey
 	 ******/
-	TEST(no_events),
+	TEST(no_events, TT_RETRIABLE),
 #endif
 	END_OF_TESTCASES
 };

test/regress_util.c

@@ -1559,12 +1559,12 @@ struct testcase_t util_testcases[] = {
 	{ "mm_malloc", test_event_malloc, 0, NULL, NULL },
 	{ "mm_calloc", test_event_calloc, 0, NULL, NULL },
 	{ "mm_strdup", test_event_strdup, 0, NULL, NULL },
-	{ "usleep", test_evutil_usleep, 0, NULL, NULL },
+	{ "usleep", test_evutil_usleep, TT_RETRIABLE, NULL, NULL },
 	{ "monotonic_res", test_evutil_monotonic_res, 0, &basic_setup, (void*)"" },
 	{ "monotonic_res_precise", test_evutil_monotonic_res, TT_OFF_BY_DEFAULT, &basic_setup, (void*)"precise" },
 	{ "monotonic_res_fallback", test_evutil_monotonic_res, TT_OFF_BY_DEFAULT, &basic_setup, (void*)"fallback" },
 	{ "monotonic_prc", test_evutil_monotonic_prc, 0, &basic_setup, (void*)"" },
-	{ "monotonic_prc_precise", test_evutil_monotonic_prc, 0, &basic_setup, (void*)"precise" },
+	{ "monotonic_prc_precise", test_evutil_monotonic_prc, TT_RETRIABLE, &basic_setup, (void*)"precise" },
 	{ "monotonic_prc_fallback", test_evutil_monotonic_prc, 0, &basic_setup, (void*)"fallback" },
 	{ "date_rfc1123", test_evutil_date_rfc1123, 0, NULL, NULL },
 	{ "evutil_v4addr_is_local", test_evutil_v4addr_is_local, 0, NULL, NULL },

test/tinytest.c

@@ -253,15 +253,12 @@ testcase_run_one(const struct testgroup_t *group,
 	}
 
 	if (outcome == OK) {
-		++n_ok;
 		if (opt_verbosity>0 && !opt_forked)
 			puts(opt_verbosity==1?"OK":"");
 	} else if (outcome == SKIP) {
-		++n_skipped;
 		if (opt_verbosity>0 && !opt_forked)
 			puts("SKIPPED");
 	} else {
-		++n_bad;
 		if (!opt_forked)
 			printf("\n  [%s FAILED]\n", testcase->name);
 	}
@@ -428,11 +425,35 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups)
 #endif
 
 	++in_tinytest_main;
-	for (i=0; groups[i].prefix; ++i)
+	for (i = 0; groups[i].prefix; ++i) {
-		for (j=0; groups[i].cases[j].name; ++j)
+		struct testgroup_t *group = &groups[i];
-			if (groups[i].cases[j].flags & TT_ENABLED_)
+		for (j = 0; group->cases[j].name; ++j) {
-				testcase_run_one(&groups[i],
+			struct testcase_t *testcase = &group->cases[j];
-						 &groups[i].cases[j]);
+			int test_attempts = 3;
+			int test_ret_err;
+
+			if (!(testcase->flags & TT_ENABLED_))
+				continue;
+
+			for (;;) {
+				test_ret_err = testcase_run_one(group, testcase);
+
+				if (test_ret_err == OK)
+					break;
+				if (!(testcase->flags & TT_RETRIABLE))
+					break;
+				printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
+				if (!test_attempts--)
+					break;
+			}
+
+			switch (test_ret_err) {
+				case OK:   ++n_ok;      break;
+				case SKIP: ++n_skipped; break;
+				default:   ++n_bad;     break;
+			}
+		}
+	}
 
 	--in_tinytest_main;
 

test/tinytest.h

@@ -34,8 +34,11 @@
 #define TT_ENABLED_  (1<<2)
 /** Flag for a test that's off by default. */
 #define TT_OFF_BY_DEFAULT  (1<<3)
+/** Flag for a test that should be runned again in case of failure (but not
+ * more then 3 times). */
+#define TT_RETRIABLE	(1<<4)
 /** If you add your own flags, make them start at this point. */
-#define TT_FIRST_USER_FLAG (1<<4)
+#define TT_FIRST_USER_FLAG (1<<5)
 
 typedef void (*testcase_fn)(void *);
 

test/tinytest_demo.c

@@ -192,6 +192,32 @@ test_timeout(void *ptr)
 	;
 }
 
+void
+test_timeout_retry(void *ptr)
+{
+	static int i = 0;
+
+	++i;
+	tt_int_op(i, !=, 1);
+
+	time_t t1, t2;
+	(void)ptr;
+	t1 = time(NULL);
+#ifdef _WIN32
+	Sleep(5000);
+#else
+	sleep(5);
+#endif
+	t2 = time(NULL);
+
+	tt_int_op(t2-t1, >=, 4);
+
+	tt_int_op(t2-t1, <=, 6);
+
+ end:
+	;
+}
+
 /* ============================================================ */
 
 /* Now we need to make sure that our tests get invoked.	  First, you take
@@ -212,6 +238,10 @@ struct testcase_t demo_tests[] = {
 	 * can enable it manually by passing +demo/timeout at the command line.*/
 	{ "timeout", test_timeout, TT_OFF_BY_DEFAULT },
 
+	/* This test will be retried. (and it will not pass from the first
+	 * time) */
+	{ "timeout_retry", test_timeout_retry, TT_RETRIABLE },
+
 	/* The array has to end with END_OF_TESTCASES. */
 	END_OF_TESTCASES
 };