# HG changeset patch
# User Kai Engert <kaie@kuix.de>
# Date 1605644505 -3600
#      Tue Nov 17 21:21:45 2020 +0100
# Node ID 8eee96dfe5184bfe9b78abb473fb5b66d408d537
# Parent  997190de89d38dccdb6de5b1e585c2f997d6d6c8
Bug 1675939 - Detect if output from GPGME.gpgme_op_decrypt_ext is armored. r=PatrickBrunschwig

diff --git a/comm/mail/extensions/openpgp/content/modules/GPGME.jsm b/comm/mail/extensions/openpgp/content/modules/GPGME.jsm
--- a/comm/mail/extensions/openpgp/content/modules/GPGME.jsm
+++ b/comm/mail/extensions/openpgp/content/modules/GPGME.jsm
@@ -74,6 +74,8 @@ var GPGME = {
       throw new Error("gpgme_new failed");
     }
 
+    GPGMELib.gpgme_set_armor(c1, 1);
+
     result.exitCode = GPGMELib.gpgme_op_decrypt_ext(
       c1,
       GPGMELib.GPGME_DECRYPT_UNWRAP,
@@ -98,14 +100,35 @@ var GPGME = {
       ).contents;
 
       // The result of decrypt(GPGME_DECRYPT_UNWRAP) is an OpenPGP message.
-      // However, GPGME always returns the results as a binary encoding.
-      // GPG 1.12.0 ignored gpgme_set_armor(context, 1) and
-      // gpgme_data_set_encoding(data_plain, GPGME_DATA_ENCODING_ARMOR).
+      // Because old versions of GPGME (e.g. 1.12.0) may return the
+      // results as a binary encoding (despite gpgme_set_armor),
+      // we check if the result looks like an armored message.
+      // If it doesn't we apply armoring ourselves.
+
+      let armor_head = "-----BEGIN PGP MESSAGE-----";
+
+      let head_of_array = ctypes.cast(
+        result_buf,
+        ctypes.char.array(armor_head.length).ptr
+      ).contents;
+
+      let isArmored = false;
 
-      // TODO: Find a way to pass the binary data directly to the
-      //       RNP.decrypt function for efficiency.
+      try {
+        // If this is binary, which usually isn't a valid UTF-8
+        // encoding, it will throw an error.
+        let head_of_array_string = head_of_array.readString();
+        if (head_of_array_string == armor_head) {
+          isArmored = true;
+        }
+      } catch (ex) {}
 
-      result.decryptedData = enArmorCB(unwrapped, result_len.value);
+      if (isArmored) {
+        result.decryptedData = unwrapped.readString();
+      } else {
+        result.decryptedData = enArmorCB(unwrapped, result_len.value);
+      }
+
       GPGMELib.gpgme_free(result_buf);
     }