--- a/m4/xiph_openssl.m4
+++ b/m4/xiph_openssl.m4
@@ -30,13 +30,15 @@ else
 fi
 
 # Now try linking to openssl
-xt_save_CFLAGS="$CFLAGS"
-xt_save_LIBS="$LIBS"
-CFLAGS="$CFLAGS $OPENSSL_CFLAGS"
-LIBS="$OPENSSL_LIBS $LIBS"
-AC_TRY_LINK([#include <openssl/ssl.h>], [void *a = SSL_new], [openssl_ok='yes'])
-CFLAGS="$xt_save_CFLAGS"
-LIBS="$xt_save_LIBS"
+if test "x$openssl_prefix" != "xno"; then
+    xt_save_CFLAGS="$CFLAGS"
+    xt_save_LIBS="$LIBS"
+    CFLAGS="$CFLAGS $OPENSSL_CFLAGS"
+    LIBS="$OPENSSL_LIBS $LIBS"
+    AC_TRY_LINK([#include <openssl/ssl.h>], [void *a = SSL_new], [openssl_ok='yes'])
+    CFLAGS="$xt_save_CFLAGS"
+    LIBS="$xt_save_LIBS"
+fi
 
 if test "$openssl_ok" = "yes"; then
     AC_DEFINE(HAVE_OPENSSL, 1, [Define if you have libopenssl.])