diff --git a/lib/jws/serializers.js b/lib/jws/serializers.js
index b42ecf527e..0688339a25 100644
--- a/lib/jws/serializers.js
+++ b/lib/jws/serializers.js
@@ -50,7 +50,7 @@ generalSerializer.validate = (jws, recipients) => {
 }
 
 const isJSON = (input) => {
-  return isObject(input) && typeof input.payload === 'string'
+  return isObject(input) && (typeof input.payload === 'string' || Buffer.isBuffer(input.payload))
 }
 
 const isValidRecipient = (recipient) => {
diff --git a/lib/jws/sign.js b/lib/jws/sign.js
index 3412152df3..9f3633361d 100644
--- a/lib/jws/sign.js
+++ b/lib/jws/sign.js
@@ -25,6 +25,7 @@ class Sign {
       payload = base64url.encode(payload)
     } else if (Buffer.isBuffer(payload)) {
       payload = base64url.encodeBuffer(payload)
+      i(this).binary = true
     } else if (isObject(payload)) {
       payload = base64url.JSON.encode(payload)
     } else {
@@ -97,13 +98,23 @@ class Sign {
         i(this).b64 = joseHeader.protected.b64
       }
       if (!joseHeader.protected.b64) {
-        i(this).payload = base64url.decode(i(this).payload)
+        if (i(this).binary) {
+          i(this).payload = base64url.decodeToBuffer(i(this).payload)
+        } else {
+          i(this).payload = base64url.decode(i(this).payload)
+        }
       }
     }
 
     recipient.header = unprotectedHeader
     recipient.protected = Object.keys(joseHeader.protected).length ? base64url.JSON.encode(joseHeader.protected) : ''
-    recipient.signature = base64url.encodeBuffer(sign(alg, key, Buffer.from(`${recipient.protected}.${i(this).payload}`)))
+
+    const toBeSigned = Buffer.concat([
+      Buffer.from(recipient.protected || ''),
+      Buffer.from('.'),
+      Buffer.isBuffer(i(this).payload) ? i(this).payload : Buffer.from(i(this).payload)
+    ])
+    recipient.signature = base64url.encodeBuffer(sign(alg, key, toBeSigned))
   }
 
   /*
diff --git a/lib/jws/verify.js b/lib/jws/verify.js
index 8433ad191a..7000e902ea 100644
--- a/lib/jws/verify.js
+++ b/lib/jws/verify.js
@@ -120,7 +120,12 @@ const jwsVerify = (skipDisjointCheck, serialization, jws, key, { crit = [], comp
 
     check(key, 'verify', alg)
 
-    if (!verify(alg, key, Buffer.from([prot, payload].join('.')), base64url.decodeToBuffer(signature))) {
+    const toBeVerified = Buffer.concat([
+      Buffer.from(prot || ''),
+      Buffer.from('.'),
+      Buffer.isBuffer(payload) ? payload : Buffer.from(payload)
+    ])
+    if (!verify(alg, key, toBeVerified, base64url.decodeToBuffer(signature))) {
       throw new errors.JWSVerificationFailed()
     }
 
diff --git a/test/jws/b64.test.js b/test/jws/b64.test.js
index d66d024f7b..a08dded0ea 100644
--- a/test/jws/b64.test.js
+++ b/test/jws/b64.test.js
@@ -1,5 +1,6 @@
 const test = require('ava')
 
+const { randomBytes } = require('crypto')
 const { JWK, JWS, errors } = require('../..')
 
 const k = JWK.asKey({
@@ -22,6 +23,13 @@ test('b64=false is supported for JWS', t => {
   t.is(JWS.verify(FIXTURE, k, { crit: ['b64'] }), FIXTURE.payload)
 })
 
+test('b64=false with buffers', t => {
+  const payload = randomBytes(32)
+  const { payload: _, ...detached } = JWS.sign.flattened(payload, k, { alg: 'HS256', b64: false, crit: ['b64'] })
+
+  t.is(JWS.verify({ ...detached, payload }, k, { crit: ['b64'] }), payload)
+})
+
 test('b64=true is also allowed', t => {
   const jws = JWS.sign.flattened('$.02', k, { alg: 'HS256', b64: true, crit: ['b64'] })
   t.is(jws.payload, 'JC4wMg')