diff --git a/src/error.rs b/src/error.rs
index 8c36f3f..969aa58 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -14,6 +14,8 @@ pub enum ServiceError {
     )]
     NoWriteAccess,
     #[error("{0}")]
+    InvalidConfig(String),
+    #[error("{0}")]
     FromString(String),
     #[error("{0}")]
     TransportError(#[from] TransportError),
diff --git a/src/fs_service/core.rs b/src/fs_service/core.rs
index 25a7399..52a8ef6 100644
--- a/src/fs_service/core.rs
+++ b/src/fs_service/core.rs
@@ -18,20 +18,22 @@ pub struct FileSystemService {
 
 impl FileSystemService {
     pub fn try_new(allowed_directories: &[String]) -> ServiceResult<Self> {
-        let normalized_dirs: Vec<PathBuf> = allowed_directories
+        let normalized_dirs: ServiceResult<Vec<PathBuf>> = allowed_directories
             .iter()
             .map(fix_dockerhub_mcp_registry_gateway)
-            .map_while(|dir| {
+            .map(|dir| {
                 let expand_result = expand_home(dir.into());
                 if !expand_result.is_dir() {
-                    panic!("{}", format!("Error: {dir} is not a directory"));
+                    return Err(ServiceError::InvalidConfig(format!(
+                        "Error: The path `{dir}` is not a valid directory. Please double-check your server configuration to ensure the directory exists and is accessible."
+                    )));
                 }
-                Some(expand_result)
+                Ok(expand_result)
             })
             .collect();
 
         Ok(Self {
-            allowed_path: RwLock::new(Arc::new(normalized_dirs)),
+            allowed_path: RwLock::new(Arc::new(normalized_dirs?)),
         })
     }
 
diff --git a/tests/test_fs_service.rs b/tests/test_fs_service.rs
index 2a18fa2..31db6ab 100644
--- a/tests/test_fs_service.rs
+++ b/tests/test_fs_service.rs
@@ -40,9 +40,10 @@ async fn test_try_new_success() {
 }
 
 #[test]
-#[should_panic(expected = "Error: /does/not/exist is not a directory")]
 fn test_try_new_invalid_directory() {
-    let _ = FileSystemService::try_new(&["/does/not/exist".to_string()]);
+    let result = FileSystemService::try_new(&["/does/not/exist".to_string()]);
+    assert!(result.is_err());
+    assert!(matches!(result, Err(ServiceError::InvalidConfig(_))));
 }
 
 #[tokio::test]