Maybe could use RLIMIT_DATA instead.

Index: src/lib/test-file-cache.c
--- src/lib/test-file-cache.c.orig
+++ src/lib/test-file-cache.c
@@ -251,6 +251,7 @@ static void test_file_cache_errors(void)
 	test_assert(size == 0);
 	test_assert(map == NULL);
 
+#ifdef HAVE_RLIMIT_AS
 	/* temporarily set a small memory limit to make mmap attempt fail */
 	struct rlimit rl_cur;
 	test_assert(getrlimit(RLIMIT_AS, &rl_cur) == 0);
@@ -274,6 +275,7 @@ static void test_file_cache_errors(void)
 	test_expect_error_string(errstr);
 	test_assert(file_cache_set_size(cache, page_size*2) == -1);
 	test_assert(setrlimit(RLIMIT_AS, &rl_cur) == 0);
+#endif
 
 	file_cache_free(&cache);
 	i_close_fd(&fd);