diff --git a/appveyor.yml b/appveyor.yml
index 18fd28fcc..4def7a287 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -73,6 +73,9 @@ test_script:
     pytest -ra --junitxml=test-results.xml
     $wc = New-Object 'System.Net.WebClient'
     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\test-results.xml))
+    If ($LastExitCode -ne 0) {
+      Throw "Tests failed"
+    }
 - dist\picard\fpcalc -version
 
 for:
@@ -95,10 +98,8 @@ for:
   build_script:
   - ps: |
       $Certificate = @(Get-ChildItem cert:\CurrentUser\My -codesign)[0]
-      if ($Certificate) {
-        & .\scripts\package\win-package-appx.ps1 -BuildNumber $env:APPVEYOR_BUILD_NUMBER -Certificate $Certificate
-        Push-AppveyorArtifact dist\*.msix
-      }
+      & .\scripts\package\win-package-appx.ps1 -BuildNumber $env:APPVEYOR_BUILD_NUMBER -Certificate $Certificate
+      Push-AppveyorArtifact dist\*.msix
 
 deploy:
 - provider: GitHub
diff --git a/scripts/package/win-common.ps1 b/scripts/package/win-common.ps1
index f62fc10a1..821f7744f 100644
--- a/scripts/package/win-common.ps1
+++ b/scripts/package/win-common.ps1
@@ -12,7 +12,8 @@ Function CodeSignBinary {
     $BinaryPath
   )
   If ($Certificate) {
-    Set-AuthenticodeSignature -FilePath $BinaryPath -Certificate $Certificate
+    Set-AuthenticodeSignature -FilePath $BinaryPath -Certificate $Certificate `
+      -ErrorAction Stop
   } Else {
     Write-Output "Skip signing $BinaryPath"
   }