diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs
index aa51df378..234f4f842 100644
--- a/src/client/io_loop.rs
+++ b/src/client/io_loop.rs
@@ -1264,7 +1264,7 @@ impl<T: InvokeUiSession> Remote<T> {
                                 self.stop_voice_call_sender = self.start_voice_call();
                             } else {
                                 // The peer refused the voice call.
-                                self.handler.on_voice_call_closed("Refused");
+                                self.handler.on_voice_call_closed("");
                             }
                         }
                     }
diff --git a/src/server/connection.rs b/src/server/connection.rs
index 1e88b9b05..7a16df811 100644
--- a/src/server/connection.rs
+++ b/src/server/connection.rs
@@ -1614,7 +1614,6 @@ impl Connection {
     pub async fn handle_voice_call(&mut self, accepted: bool) {
         if let Some(ts) = self.voice_call_request_timestamp.take() {
             let msg = new_voice_call_response(ts.get(), accepted);
-            self.send(msg).await;
             if accepted {
                 // Backup the default input device.
                 let audio_input_device = Config::get_option("audio-input");
@@ -1625,7 +1624,10 @@ impl Connection {
                     set_sound_input(device);
                 }
                 self.send_to_cm(Data::StartVoiceCall);
+            } else {
+                self.send_to_cm(Data::CloseVoiceCall("".to_owned()));
             }
+            self.send(msg).await;
         } else {
             log::warn!("Possible a voice call attack.");
         }