/hg/release/icedtea7-forest-2.4/jdk: 3 new changesets
andrew at icedtea.classpath.org
andrew at icedtea.classpath.org
Thu May 22 17:33:18 UTC 2014
changeset b4fc2a92ae21 in /hg/release/icedtea7-forest-2.4/jdk
details: http://icedtea.classpath.org/hg/release/icedtea7-forest-2.4/jdk?cmd=changeset;node=b4fc2a92ae21
author: andrew
date: Wed May 21 15:25:53 2014 +0100
PR1781: NSS PKCS11 provider fails to handle multipart AES encryption
changeset a982f3aa40ed in /hg/release/icedtea7-forest-2.4/jdk
details: http://icedtea.classpath.org/hg/release/icedtea7-forest-2.4/jdk?cmd=changeset;node=a982f3aa40ed
author: andrew
date: Wed May 21 15:52:15 2014 +0100
RH905128: [CRASH] OpenJDK-1.7.0 while using NSS security provider and kerberos
changeset 521761df8dea in /hg/release/icedtea7-forest-2.4/jdk
details: http://icedtea.classpath.org/hg/release/icedtea7-forest-2.4/jdk?cmd=changeset;node=521761df8dea
author: xuelei
date: Mon Aug 12 02:34:25 2013 -0700
8013809: deadlock in SSLSocketImpl between between write and close
Reviewed-by: wetmore
diffstat:
src/share/classes/sun/security/pkcs11/P11Cipher.java | 388 +++++++++++------
src/share/classes/sun/security/ssl/SSLSocketImpl.java | 4 +-
2 files changed, 241 insertions(+), 151 deletions(-)
diffs (truncated from 572 to 500 lines):
diff -r c8c15663cef6 -r 521761df8dea src/share/classes/sun/security/pkcs11/P11Cipher.java
--- a/src/share/classes/sun/security/pkcs11/P11Cipher.java Thu Apr 17 17:20:13 2014 +0100
+++ b/src/share/classes/sun/security/pkcs11/P11Cipher.java Mon Aug 12 02:34:25 2013 -0700
@@ -69,7 +69,7 @@
private static interface Padding {
// ENC: format the specified buffer with padding bytes and return the
// actual padding length
- int setPaddingBytes(byte[] paddingBuffer, int padLen);
+ int setPaddingBytes(byte[] paddingBuffer, int offset, int padLen);
// DEC: return the length of trailing padding bytes given the specified
// padded data
@@ -90,8 +90,8 @@
this.blockSize = blockSize;
}
- public int setPaddingBytes(byte[] paddingBuffer, int padLen) {
- Arrays.fill(paddingBuffer, 0, padLen, (byte) (padLen & 0x007f));
+ public int setPaddingBytes(byte[] paddingBuffer, int offset, int padLen) {
+ Arrays.fill(paddingBuffer, offset, offset + padLen, (byte) (padLen & 0x007f));
return padLen;
}
@@ -160,10 +160,16 @@
// original IV, if in MODE_CBC or MODE_CTR
private byte[] iv;
- // number of bytes buffered internally by the native mechanism and padBuffer
- // if we do the padding
+ // number of bytes buffered by the blockBuffer
private int bytesBuffered;
+ // number of bytes buffered internally
+ private int bytesBufferedInt;
+
+ // bytes buffered from an incomplete block
+ private byte[] blockBuffer;
+ private int blockBufferLen;
+
P11Cipher(Token token, String algorithm, long mechanism)
throws PKCS11Exception, NoSuchAlgorithmException {
super();
@@ -194,6 +200,9 @@
// should not happen
throw new ProviderException(nspe);
}
+
+ if (blockSize > 0)
+ blockBuffer = new byte[blockSize];
}
protected void engineSetMode(String mode) throws NoSuchAlgorithmException {
@@ -403,7 +412,8 @@
token.p11.C_DecryptFinal(session.id(), 0, buffer, 0, bufLen);
}
} catch (PKCS11Exception e) {
- throw new ProviderException("Cancel failed", e);
+ if (e.getErrorCode() != CKR_OPERATION_NOT_INITIALIZED)
+ throw new ProviderException("Cancel failed", e);
} finally {
reset();
}
@@ -435,7 +445,9 @@
throw ex;
}
bytesBuffered = 0;
+ bytesBufferedInt = 0;
padBufferLen = 0;
+ blockBufferLen = 0;
initialized = true;
}
@@ -445,7 +457,7 @@
return 0;
}
- int result = inLen + bytesBuffered;
+ int result = inLen + bytesBuffered + bytesBufferedInt;
if (blockSize != 0) {
// minus the number of bytes in the last incomplete block.
result -= (result & (blockSize - 1));
@@ -459,7 +471,7 @@
return 0;
}
- int result = inLen + bytesBuffered;
+ int result = inLen + bytesBuffered + bytesBufferedInt;
if (blockSize != 0 && encrypt && paddingType != PAD_NONE) {
// add the number of bytes to make the last block complete.
result += (blockSize - (result & (blockSize - 1)));
@@ -471,7 +483,9 @@
private void reset() {
initialized = false;
bytesBuffered = 0;
+ bytesBufferedInt = 0;
padBufferLen = 0;
+ blockBufferLen = 0;
if (session != null) {
session = token.releaseSession(session);
}
@@ -546,50 +560,65 @@
}
try {
ensureInitialized();
- int k = 0;
- if (encrypt) {
- k = token.p11.C_EncryptUpdate(session.id(), 0, in, inOfs, inLen,
- 0, out, outOfs, outLen);
- } else {
- int newPadBufferLen = 0;
- if (paddingObj != null) {
- if (padBufferLen != 0) {
- // NSS throws up when called with data not in multiple
- // of blocks. Try to work around this by holding the
- // extra data in padBuffer.
- if (padBufferLen != padBuffer.length) {
- int bufCapacity = padBuffer.length - padBufferLen;
- if (inLen > bufCapacity) {
- bufferInputBytes(in, inOfs, bufCapacity);
- inOfs += bufCapacity;
- inLen -= bufCapacity;
- } else {
- bufferInputBytes(in, inOfs, inLen);
- return 0;
- }
- }
- k = token.p11.C_DecryptUpdate(session.id(),
- 0, padBuffer, 0, padBufferLen,
- 0, out, outOfs, outLen);
- padBufferLen = 0;
- }
- newPadBufferLen = inLen & (blockSize - 1);
- if (newPadBufferLen == 0) {
- newPadBufferLen = padBuffer.length;
- }
- inLen -= newPadBufferLen;
- }
- if (inLen > 0) {
- k += token.p11.C_DecryptUpdate(session.id(), 0, in, inOfs,
- inLen, 0, out, (outOfs + k), (outLen - k));
- }
- // update 'padBuffer' if using our own padding impl.
- if (paddingObj != null) {
- bufferInputBytes(in, inOfs + inLen, newPadBufferLen);
- }
- }
- bytesBuffered += (inLen - k);
- return k;
+ int bufRes = 0;
+ int inRes = 0;
+ int newBlockBufferLen = 0;
+
+ // NSS throws up when called with data not in multiple
+ // of blocks. Try to work around this by holding the
+ // extra data in blockBuffer.
+ if (blockBufferLen != 0) {
+ if (blockBufferLen != blockBuffer.length) {
+ int bufCapacity = blockBuffer.length - blockBufferLen;
+ if (inLen >= bufCapacity) {
+ bufferInputBytes(in, inOfs, bufCapacity);
+ inOfs += bufCapacity;
+ inLen -= bufCapacity;
+ } else {
+ bufferInputBytes(in, inOfs, inLen);
+ return 0;
+ }
+ }
+ if (encrypt) {
+ bufRes = token.p11.C_EncryptUpdate(session.id(), 0, blockBuffer, 0,
+ blockBufferLen, 0, out, outOfs,
+ outLen);
+ } else {
+ bufRes = token.p11.C_DecryptUpdate(session.id(), 0, blockBuffer, 0,
+ blockBufferLen, 0, out, outOfs,
+ outLen);
+ }
+ bytesBufferedInt += (blockBufferLen - bufRes);
+ blockBufferLen = 0;
+ bytesBuffered = 0;
+ }
+
+ if (inLen == 0)
+ return bufRes;
+
+ if (blockBuffer != null) {
+ newBlockBufferLen = inLen & (blockSize - 1);
+ if (!encrypt && paddingObj != null && newBlockBufferLen == 0)
+ // Hold the last block in the buffer if we need to unpad
+ newBlockBufferLen = blockBuffer.length;
+ inLen -= newBlockBufferLen;
+ bufferInputBytes(in, inOfs + inLen, newBlockBufferLen);
+ }
+
+ if (inLen > 0) {
+ if (encrypt) {
+ inRes = token.p11.C_EncryptUpdate(session.id(), 0, in, inOfs,
+ inLen, 0, out, (outOfs + bufRes),
+ (outLen - bufRes));
+ } else {
+ inRes = token.p11.C_DecryptUpdate(session.id(), 0, in, inOfs,
+ inLen, 0, out, (outOfs + bufRes),
+ (outLen - bufRes));
+ }
+ bytesBufferedInt += (inLen - inRes);
+ }
+
+ return inRes + bufRes;
} catch (PKCS11Exception e) {
if (e.getErrorCode() == CKR_BUFFER_TOO_SMALL) {
throw (ShortBufferException)
@@ -642,70 +671,79 @@
}
}
- int k = 0;
- if (encrypt) {
- if (inAddr == 0 && inArray == null) {
- inArray = new byte[inLen];
- inBuffer.get(inArray);
- } else {
- inBuffer.position(origPos + inLen);
- }
- k = token.p11.C_EncryptUpdate(session.id(),
- inAddr, inArray, inOfs, inLen,
- outAddr, outArray, outOfs, outLen);
- } else {
- int newPadBufferLen = 0;
- if (paddingObj != null) {
- if (padBufferLen != 0) {
- // NSS throws up when called with data not in multiple
- // of blocks. Try to work around this by holding the
- // extra data in padBuffer.
- if (padBufferLen != padBuffer.length) {
- int bufCapacity = padBuffer.length - padBufferLen;
- if (inLen > bufCapacity) {
- bufferInputBytes(inBuffer, bufCapacity);
- inOfs += bufCapacity;
- inLen -= bufCapacity;
- } else {
- bufferInputBytes(inBuffer, inLen);
- return 0;
- }
- }
- k = token.p11.C_DecryptUpdate(session.id(), 0,
- padBuffer, 0, padBufferLen, outAddr, outArray,
- outOfs, outLen);
- padBufferLen = 0;
- }
- newPadBufferLen = inLen & (blockSize - 1);
- if (newPadBufferLen == 0) {
- newPadBufferLen = padBuffer.length;
- }
- inLen -= newPadBufferLen;
- }
- if (inLen > 0) {
- if (inAddr == 0 && inArray == null) {
- inArray = new byte[inLen];
- inBuffer.get(inArray);
- } else {
- inBuffer.position(inBuffer.position() + inLen);
- }
- k += token.p11.C_DecryptUpdate(session.id(), inAddr,
- inArray, inOfs, inLen, outAddr, outArray,
- (outOfs + k), (outLen - k));
- }
- // update 'padBuffer' if using our own padding impl.
- if (paddingObj != null && newPadBufferLen != 0) {
- bufferInputBytes(inBuffer, newPadBufferLen);
- }
- }
- bytesBuffered += (inLen - k);
+ int bufRes = 0;
+ int inRes = 0;
+ int newBlockBufferLen = 0;
+
+ // NSS throws up when called with data not in multiple
+ // of blocks. Try to work around this by holding the
+ // extra data in blockBuffer.
+ if (blockBufferLen != 0) {
+ if (blockBufferLen != blockBuffer.length) {
+ int bufCapacity = blockBuffer.length - blockBufferLen;
+ if (inLen >= bufCapacity) {
+ bufferInputBytes(inBuffer, bufCapacity);
+ inOfs += bufCapacity;
+ inLen -= bufCapacity;
+ } else {
+ bufferInputBytes(inBuffer, inLen);
+ return 0;
+ }
+ }
+ if (encrypt) {
+ bufRes = token.p11.C_EncryptUpdate(session.id(), 0, blockBuffer, 0,
+ blockBufferLen, outAddr, outArray, outOfs,
+ outLen);
+ } else {
+ bufRes = token.p11.C_DecryptUpdate(session.id(), 0, blockBuffer, 0,
+ blockBufferLen, outAddr, outArray, outOfs,
+ outLen);
+ }
+ bytesBufferedInt += (blockBufferLen - bufRes);
+ blockBufferLen = 0;
+ bytesBuffered = 0;
+ }
+
+ if (inLen == 0)
+ return bufRes;
+
+ if (blockBuffer != null) {
+ newBlockBufferLen = inLen & (blockSize - 1);
+ if (!encrypt && paddingObj != null && newBlockBufferLen == 0)
+ // Hold the last block in the buffer if we need to unpad
+ newBlockBufferLen = blockBuffer.length;
+ inLen -= newBlockBufferLen;
+ bufferInputBytes(inBuffer, newBlockBufferLen);
+ }
+
+ if (inAddr == 0 && inArray == null) {
+ inArray = new byte[inLen];
+ inBuffer.get(inArray);
+ } else {
+ inBuffer.position(inBuffer.position() + inLen);
+ }
+
+ if (inLen > 0) {
+ if (encrypt) {
+ inRes = token.p11.C_EncryptUpdate(session.id(), inAddr, inArray, inOfs,
+ inLen, outAddr, outArray, (outOfs + bufRes),
+ (outLen - bufRes));
+ } else {
+ inRes = token.p11.C_DecryptUpdate(session.id(), inAddr, inArray, inOfs,
+ inLen, outAddr, outArray, (outOfs + bufRes),
+ (outLen - bufRes));
+ }
+ bytesBufferedInt += (inLen - inRes);
+ }
+
+ int total = inRes + bufRes;
if (!(outBuffer instanceof DirectBuffer) &&
!outBuffer.hasArray()) {
- outBuffer.put(outArray, outOfs, k);
+ outBuffer.put(outArray, outOfs, total);
} else {
- outBuffer.position(outBuffer.position() + k);
+ outBuffer.position(outBuffer.position() + total);
}
- return k;
+ return total;
} catch (PKCS11Exception e) {
// Reset input buffer to its original position for
inBuffer.position(origPos);
@@ -722,45 +760,71 @@
throws ShortBufferException, IllegalBlockSizeException,
BadPaddingException {
int requiredOutLen = doFinalLength(0);
+ boolean updating = false;
+ PKCS11Exception err = null;
+
if (outLen < requiredOutLen) {
throw new ShortBufferException();
}
try {
ensureInitialized();
int k = 0;
- if (encrypt) {
- if (paddingObj != null) {
- int actualPadLen = paddingObj.setPaddingBytes(padBuffer,
- requiredOutLen - bytesBuffered);
- k = token.p11.C_EncryptUpdate(session.id(),
- 0, padBuffer, 0, actualPadLen,
- 0, out, outOfs, outLen);
- }
+ if (encrypt) {
+ if (blockBuffer != null) {
+ // Do we need to pad?
+ if (paddingObj != null) {
+ int actualPadLen = paddingObj.setPaddingBytes(blockBuffer,
+ blockBufferLen, blockSize - blockBufferLen);
+ blockBufferLen = blockSize;
+ }
+ updating = true;
+ k = token.p11.C_EncryptUpdate(session.id(),
+ 0, blockBuffer, 0, blockBufferLen,
+ 0, out, outOfs, outLen);
+ updating = false;
+ }
k += token.p11.C_EncryptFinal(session.id(),
0, out, (outOfs + k), (outLen - k));
} else {
+ if (blockBufferLen != 0) {
+ if (paddingObj == null)
+ k = token.p11.C_DecryptUpdate(session.id(), 0,
+ blockBuffer, 0, blockBufferLen, 0,
+ out, outOfs, outLen);
+ else
+ k = token.p11.C_DecryptUpdate(session.id(), 0,
+ blockBuffer, 0, blockBufferLen, 0,
+ padBuffer, 0, padBuffer.length);
+ }
if (paddingObj != null) {
- if (padBufferLen != 0) {
- k = token.p11.C_DecryptUpdate(session.id(), 0,
- padBuffer, 0, padBufferLen, 0, padBuffer, 0,
- padBuffer.length);
- }
k += token.p11.C_DecryptFinal(session.id(), 0, padBuffer, k,
padBuffer.length - k);
int actualPadLen = paddingObj.unpad(padBuffer, k);
k -= actualPadLen;
System.arraycopy(padBuffer, 0, out, outOfs, k);
} else {
- k = token.p11.C_DecryptFinal(session.id(), 0, out, outOfs,
- outLen);
+ k += token.p11.C_DecryptFinal(session.id(), 0, out, (outOfs + k),
+ (outLen - k));
}
}
return k;
} catch (PKCS11Exception e) {
+ err = e;
handleException(e);
throw new ProviderException("doFinal() failed", e);
} finally {
- reset();
+ if (err != null) {
+ if (err.getErrorCode() != CKR_BUFFER_TOO_SMALL) {
+ if (updating)
+ // Work around NSS not cancelling the
+ // operation on an error in update
+ cancelOperation();
+ else
+ reset();
+ }
+ } else {
+ reset();
+ }
}
}
@@ -769,6 +833,9 @@
BadPaddingException {
int outLen = outBuffer.remaining();
int requiredOutLen = doFinalLength(0);
+ boolean updating = false;
+ PKCS11Exception err = null;
+
if (outLen < requiredOutLen) {
throw new ShortBufferException();
}
@@ -793,24 +860,34 @@
int k = 0;
- if (encrypt) {
- if (paddingObj != null) {
- int actualPadLen = paddingObj.setPaddingBytes(padBuffer,
- requiredOutLen - bytesBuffered);
- k = token.p11.C_EncryptUpdate(session.id(),
- 0, padBuffer, 0, actualPadLen,
- outAddr, outArray, outOfs, outLen);
- }
+ if (encrypt) {
+ if (blockBuffer != null) {
+ // Do we need to pad?
+ if (paddingObj != null) {
+ int actualPadLen = paddingObj.setPaddingBytes(blockBuffer,
+ blockBufferLen, blockSize - blockBufferLen);
+ blockBufferLen = blockSize;
+ }
+ updating = true;
+ k = token.p11.C_EncryptUpdate(session.id(),
+ 0, blockBuffer, 0, blockBufferLen,
+ outAddr, outArray, outOfs, outLen);
+ updating = false;
+ }
k += token.p11.C_EncryptFinal(session.id(),
outAddr, outArray, (outOfs + k), (outLen - k));
- } else {
+ } else {
+ if (blockBufferLen != 0) {
+ k = token.p11.C_DecryptUpdate(session.id(), 0,
+ blockBuffer, 0, blockBufferLen, outAddr,
+ outArray, outOfs, outLen);
+ } else {
+ k = token.p11.C_DecryptUpdate(session.id(), 0,
+ blockBuffer, 0, blockBufferLen, 0,
+ padBuffer, 0, padBuffer.length);
+ }
+
if (paddingObj != null) {
- if (padBufferLen != 0) {
- k = token.p11.C_DecryptUpdate(session.id(),
- 0, padBuffer, 0, padBufferLen,
- 0, padBuffer, 0, padBuffer.length);
- padBufferLen = 0;
- }
k += token.p11.C_DecryptFinal(session.id(),
0, padBuffer, k, padBuffer.length - k);
int actualPadLen = paddingObj.unpad(padBuffer, k);
@@ -819,7 +896,8 @@
outOfs = 0;
More information about the distro-pkg-dev
mailing list