diff --git a/cmd/justcaptchad/main.go b/cmd/justcaptchad/main.go
index 4757f12..a422de6 100644
--- a/cmd/justcaptchad/main.go
+++ b/cmd/justcaptchad/main.go
@@ -28,10 +28,6 @@ func main() {
 		return
 	}
 
-	if !strings.ContainsRune(*listenAddress, ':') {
-		defer os.Remove(*listenAddress)
-	}
-
 	captcha.Init(*captchaExpire)
 
 	hand := handlers.New(*captchaExpire)
@@ -45,6 +41,7 @@ func main() {
 	var network string
 	if !strings.ContainsRune(*listenAddress, ':') {
 		network = "unix"
+		defer os.Remove(*listenAddress)
 	} else {
 		ap, err := netip.ParseAddrPort(*listenAddress)
 		if err != nil {