diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py
index d6cd0774a8..d35dc0e215 100644
--- a/synapse/storage/databases/main/user_directory.py
+++ b/synapse/storage/databases/main/user_directory.py
@@ -42,7 +42,7 @@ try:
     import icu
 
     USE_ICU = True
-except ModuleNotFoundError:
+except (ModuleNotFoundError, ImportError):
     USE_ICU = False
 
 from synapse.api.errors import StoreError