[for-fun][icedtea-web] curl integration
Deepak Bhole
dbhole at redhat.com
Tue Apr 29 14:37:14 UTC 2014
* Jiri Vanek <jvanek at redhat.com> [2014-04-29 08:42]:
> I was facing various bugs on https, and I have not seen way out.
>
> And I needed to workaround issues after, the jars are done, so this evil was done .. :)
>
> This patch is - after pure-java fails to download - trying curl the
> resource (by forking insecure curl process).
>
> Well it is working pretty fine :) (!!the security prompt is missing!!)
>
> Now i was enlighten - by Omair - and the workaround with SSLv2
> handshake together with "trust all" (after a lot of security
> confirmations) temporal manager. So I will rather play with those
> two options. It should work for most cases. Anyway - this curl patch
> may be safety rope once everything else fails:)
>
>
> J.
I'm not sure if calling external processes from a plugin (other than
java :)) is a good idea.
. For one, this is a Linux only solution and while we do not support
Windows, I would like for us to not go out of our way to break it.
. It adds a dependency on curl
Is there no way to use a pure Java solution?
Deepak
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/CURL.java
> --- /dev/null Thu Jan 01 00:00:00 1970 +0000
> +++ b/netx/net/sourceforge/jnlp/cache/CURL.java Tue Apr 29 13:48:19 2014 +0200
> @@ -0,0 +1,43 @@
> +package net.sourceforge.jnlp.cache;
> +
> +import java.io.IOException;
> +import java.net.URL;
> +import java.net.URLConnection;
> +
> +
> +public class CURL {
> +
> + public final URL url;
> + public final boolean curl;
> + private CURLConnection fakeConnection;
> +
> + public CURL(URL u, boolean c) {
> + url = u;
> + curl = c;
> + }
> +
> + public CURL(URL u) {
> + url = u;
> + curl = false;
> + }
> +
> + public URL getUrl() {
> + return url;
> + }
> +
> + public boolean isCurl() {
> + return curl;
> + }
> +
> + URLConnection openConnection() throws IOException {
> + if (isCurl()){
> + if (fakeConnection == null){
> + fakeConnection = new CURLConnection(url);
> + }
> + return fakeConnection;
> + } else {
> + return url.openConnection();
> + }
> + }
> +
> +}
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/CURLConnection.java
> --- /dev/null Thu Jan 01 00:00:00 1970 +0000
> +++ b/netx/net/sourceforge/jnlp/cache/CURLConnection.java Tue Apr 29 13:48:19 2014 +0200
> @@ -0,0 +1,136 @@
> +package net.sourceforge.jnlp.cache;
> +
> +import java.io.BufferedInputStream;
> +import java.io.BufferedReader;
> +import java.io.FileOutputStream;
> +import java.io.IOException;
> +import java.io.InputStream;
> +import java.io.InputStreamReader;
> +import java.io.OutputStream;
> +import java.net.URL;
> +import java.net.URLConnection;
> +import java.text.SimpleDateFormat;
> +import java.util.ArrayList;
> +import java.util.Date;
> +import java.util.List;
> +
> +
> +public class CURLConnection extends URLConnection{
> +
> + private final URL setUrl;
> + private List<String> header;
> + private Integer length;
> + private SimpleDateFormat format = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
> + private Date date ;
> +
> + public CURLConnection(URL u) {
> + super(u);
> + this.setUrl = u;
> + }
> +
> + public List<String> getHeader() throws IOException {
> + if (header == null) {
> + ProcessBuilder pb = new ProcessBuilder("curl", "-Ik", setUrl.toExternalForm());
> + Process p = pb.start();
> + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
> + header = new ArrayList<>();
> + while (true) {
> + String s = r.readLine();
> + if (s == null) {
> + break;
> + }
> + header.add(s.trim());
> + }
> + }
> + return header;
> + }
> +
> + public int getLength() {
> + if (length != null) {
> + return length;
> + }
> + try {
> + List<String> l = getHeader();
> + for (String s : l) {
> + if (s.toLowerCase().startsWith("content-length:")) {
> + String sl = s.split(":\\s*")[1];
> + length = new Integer(sl);
> + return length;
> + }
> + }
> + return 0;
> + } catch (Exception ex) {
> + return 0;
> + }
> +
> + }
> +
> + @Override
> + public long getLastModified() {
> + return getLastModifiedDate().getTime();
> + }
> +
> +
> + private static final String lastModified="Last-Modified:".toLowerCase();
> + public Date getLastModifiedDate() {
> + if (date != null) {
> + return date;
> + }
> + try {
> + List<String> l = getHeader();
> + for (String s : l) {
> + if (s.toLowerCase().startsWith(lastModified)) {
> + String sl = s.substring(lastModified.length()).trim();
> + date = date = format.parse(sl);
> + return date;
> + }
> + }
> + return new Date();
> + } catch (Exception ex) {
> + return new Date();
> + }
> +
> + }
> +
> +
> + public InputStream read() throws IOException {
> + ProcessBuilder pb = new ProcessBuilder("curl", "-k", setUrl.toExternalForm());
> + Process p = pb.start();
> + BufferedInputStream r = new BufferedInputStream(p.getInputStream());
> + return r;
> +
> + }
> +
> + @Override
> + public InputStream getInputStream() throws IOException {
> + return read();
> + }
> +
> +
> + public static void main(String[] args) throws IOException {
> + CURLConnection crl = new CURLConnection(new URL("https://rholm.corp.redhat.com:4443/contentstorage/Adapter/browserTest.jar"));
> + System.out.println(crl.getLength());
> + System.out.println(crl.getLastModified());
> + InputStream input = crl.read();
> + OutputStream output = new FileOutputStream("/home/jvanek/Desktop/jar.jar");
> + byte[] buffer = new byte[256];
> + int bytesRead = 0;
> + while ((bytesRead = input.read(buffer)) != -1) {
> + output.write(buffer, 0, bytesRead);
> + }
> + }
> +
> + @Override
> + public void connect() throws IOException {
> +
> + }
> +
> + @Override
> + public int getContentLength() {
> + return getLength();
> + }
> +
> +
> +
> +
> +}
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/CacheEntry.java
> --- a/netx/net/sourceforge/jnlp/cache/CacheEntry.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/netx/net/sourceforge/jnlp/cache/CacheEntry.java Tue Apr 29 13:48:19 2014 +0200
> @@ -64,9 +64,9 @@
> * Initialize the cache entry data from a connection to the
> * remote resource (does not store data).
> */
> - void initialize(URLConnection connection) {
> - long modified = connection.getLastModified();
> - long length = connection.getContentLength(); // an int
> + void initialize(long lastModified, int ilength) {
> + long modified = lastModified;
> + long length = ilength; // an int
>
> properties.setProperty("content-length", Long.toString(length));
> properties.setProperty("last-modified", Long.toString(modified));
> @@ -107,14 +107,14 @@
> * @param connection a connection to the remote URL
> * @return whether the cache contains the version
> */
> - public boolean isCurrent(URLConnection connection) {
> + public boolean isCurrent(long lastModified) {
> boolean cached = isCached();
>
> if (!cached)
> return false;
>
> try {
> - long remoteModified = connection.getLastModified();
> + long remoteModified = lastModified;
> long cachedModified = Long.parseLong(properties.getProperty("last-modified"));
>
> if (remoteModified > 0 && remoteModified <= cachedModified)
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/CacheUtil.java
> --- a/netx/net/sourceforge/jnlp/cache/CacheUtil.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/netx/net/sourceforge/jnlp/cache/CacheUtil.java Tue Apr 29 13:48:19 2014 +0200
> @@ -258,7 +258,7 @@
> connection.connect();
>
> CacheEntry entry = new CacheEntry(source, version); // could pool this
> - boolean result = entry.isCurrent(connection);
> + boolean result = entry.isCurrent(connection.getLastModified());
>
> OutputController.getLogger().log("isCurrent: " + source + " = " + result);
>
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/Resource.java
> --- a/netx/net/sourceforge/jnlp/cache/Resource.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/netx/net/sourceforge/jnlp/cache/Resource.java Tue Apr 29 13:48:19 2014 +0200
> @@ -72,7 +72,7 @@
> URL location;
>
> /** the location to use when downloading */
> - private URL downloadLocation;
> + private CURL downloadLocation;
>
> /** the local file downloaded to */
> File localFile;
> @@ -98,8 +98,8 @@
> /**
> * Create a resource.
> */
> - private Resource(URL location, Version requestVersion, UpdatePolicy updatePolicy) {
> - this.location = location;
> + private Resource(CURL location, Version requestVersion, UpdatePolicy updatePolicy) {
> + this.location = location.getUrl();
> this.downloadLocation = location;
> this.requestVersion = requestVersion;
> this.updatePolicy = updatePolicy;
> @@ -109,7 +109,7 @@
> * Return a shared Resource object representing the given
> * location and version.
> */
> - public static Resource getResource(URL location, Version requestVersion, UpdatePolicy updatePolicy) {
> + public static Resource getResource(CURL location, Version requestVersion, UpdatePolicy updatePolicy) {
> synchronized (resources) {
> Resource resource = new Resource(location, requestVersion, updatePolicy);
>
> @@ -142,7 +142,7 @@
> * file name to support versioning and compression
> * @return the url to use when downloading
> */
> - public URL getDownloadLocation() {
> + public CURL getDownloadLocation() {
> return downloadLocation;
> }
>
> @@ -150,7 +150,7 @@
> * Set the url to use for downloading the resource
> * @param location
> */
> - public void setDownloadLocation(URL location) {
> + public void setDownloadLocation(CURL location) {
> downloadLocation = location;
> }
>
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/cache/ResourceTracker.java
> --- a/netx/net/sourceforge/jnlp/cache/ResourceTracker.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/netx/net/sourceforge/jnlp/cache/ResourceTracker.java Tue Apr 29 13:48:19 2014 +0200
> @@ -189,7 +189,7 @@
> OutputController.getLogger().log(OutputController.Level.ERROR_ALL, "Normalization of " + location.toString() + " have failed");
> OutputController.getLogger().log(ex);
> }
> - Resource resource = Resource.getResource(location, version, updatePolicy);
> + Resource resource = Resource.getResource(new CURL(location), version, updatePolicy);
> boolean downloaded = false;
>
> synchronized (resources) {
> @@ -653,7 +653,7 @@
>
> try {
> // create out second in case in does not exist
> - URL realLocation = resource.getDownloadLocation();
> + CURL realLocation = resource.getDownloadLocation();
> URLConnection con = realLocation.openConnection();
> con.addRequestProperty("Accept-Encoding", "pack200-gzip, gzip");
>
> @@ -672,7 +672,7 @@
> realLocation + " (encoding : " + contentEncoding + ")");
>
> boolean packgz = "pack200-gzip".equals(contentEncoding) ||
> - realLocation.getPath().endsWith(".pack.gz");
> + realLocation.getUrl().getPath().endsWith(".pack.gz");
> boolean gzip = "gzip".equals(contentEncoding);
>
> // It's important to check packgz first. If a stream is both
> @@ -689,7 +689,7 @@
> CacheEntry downloadEntry = new CacheEntry(downloadLocation, resource.downloadVersion);
> File finalFile = CacheUtil.getCacheFile(resource.location, resource.downloadVersion); // This is where extracted version will be, or downloaded file if not compressed.
>
> - if (!downloadEntry.isCurrent(con)) {
> + if (!downloadEntry.isCurrent(con.getLastModified())) {
> // Make sure we don't re-download the file. however it will wait as if it was downloading.
> // (This is fine because file is not ready yet anyways)
> byte buf[] = new byte[1024];
> @@ -714,7 +714,7 @@
> * If the file was compressed, uncompress it.
> */
> if (packgz) {
> - downloadEntry.initialize(con);
> + downloadEntry.initialize(con.getLastModified(), con.getContentLength());
> GZIPInputStream gzInputStream = new GZIPInputStream(new FileInputStream(CacheUtil
> .getCacheFile(downloadLocation, resource.downloadVersion)));
> InputStream inputStream = new BufferedInputStream(gzInputStream);
> @@ -729,7 +729,7 @@
> inputStream.close();
> gzInputStream.close();
> } else if (gzip) {
> - downloadEntry.initialize(con);
> + downloadEntry.initialize(con.getLastModified(), con.getContentLength());
> GZIPInputStream gzInputStream = new GZIPInputStream(new FileInputStream(CacheUtil
> .getCacheFile(downloadLocation, resource.downloadVersion)));
> InputStream inputStream = new BufferedInputStream(gzInputStream);
> @@ -785,8 +785,31 @@
> File localFile = CacheUtil.getCacheFile(resource.location, resource.downloadVersion);
>
> // connect
> - URL finalLocation = findBestUrl(resource);
> + CURL finalLocation = findBestUrl(resource);
>
> +
> + if (finalLocation == null) {
> + DownloadOptions options = downloadOptions.get(resource);
> + if (options == null) {
> + options = new DownloadOptions(false, false);
> + }
> + OutputController.getLogger().log(OutputController.Level.ERROR_ALL, "Attempted to download " + resource.location + ", failed, trying curl");
> + List<URL> urls = new ResourceUrlCreator(resource, options).getUrls();
> + for (URL url : urls) {
> + try {
> + CURLConnection c = new CURLConnection(url);
> + int l = c.getLength();
> + if (l > 0) {
> + finalLocation = new CURL(url, true);
> + OutputController.getLogger().log(OutputController.Level.MESSAGE_ALL, "Curl succeeded on " + url.toExternalForm() + ", using");
> + break;
> + }
> + } catch (Exception ex) {
> + OutputController.getLogger().log(OutputController.Level.ERROR_DEBUG, ex);
> + }
> + }
> + }
> +
> if (finalLocation == null) {
> OutputController.getLogger().log(OutputController.Level.ERROR_ALL, "Attempted to download " + resource.location + ", but failed to connect!");
> throw new NullPointerException("finalLocation == null"); // Caught below
> @@ -823,8 +846,9 @@
> }
>
> // update cache entry
> - if (!current)
> - entry.initialize(connection);
> + if (!current) {
> + entry.initialize(connection.getLastModified(), connection.getContentLength());
> + }
>
> entry.setLastUpdated(System.currentTimeMillis());
> entry.store();
> @@ -935,7 +959,7 @@
> * @param resource the resource
> * @return the best URL, or null if all failed to resolve
> */
> - URL findBestUrl(Resource resource) {
> + CURL findBestUrl(Resource resource) {
> DownloadOptions options = downloadOptions.get(resource);
> if (options == null) {
> options = new DownloadOptions(false, false);
> @@ -967,9 +991,10 @@
> OutputController.getLogger().log(OutputController.Level.ERROR_DEBUG, "For " + resource.toString() + " the server returned " + response.result + " code for " + requestMethod + " request for " + url.toExternalForm());
> } else {
> OutputController.getLogger().log(OutputController.Level.ERROR_DEBUG, "best url for " + resource.toString() + " is " + url.toString() + " by " + requestMethod);
> - return url; /* This is the best URL */
> + return new CURL(url); /* This is the best URL */
> }
> - } catch (IOException e) {
> + //there can be various SSL exceptions which we cna survive
> + } catch (Exception e) {
> // continue to next candidate
> OutputController.getLogger().log(OutputController.Level.ERROR_DEBUG, "While processing " + url.toString() + " by " + requestMethod + " for resource " + resource.toString() + " got " + e + ": ");
> OutputController.getLogger().log(e);
> diff -r 3f9913affb06 netx/net/sourceforge/jnlp/security/VariableX509TrustManagerJDK7.java
> --- a/netx/net/sourceforge/jnlp/security/VariableX509TrustManagerJDK7.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/netx/net/sourceforge/jnlp/security/VariableX509TrustManagerJDK7.java Tue Apr 29 13:48:19 2014 +0200
> @@ -37,14 +37,11 @@
>
> package net.sourceforge.jnlp.security;
>
> -import java.lang.reflect.InvocationTargetException;
> -import java.lang.reflect.Method;
> import java.net.Socket;
> import java.security.cert.CertificateException;
> import java.security.cert.X509Certificate;
>
> import javax.net.ssl.SSLEngine;
> -import javax.net.ssl.SSLSession;
> import javax.net.ssl.SSLSocket;
> import javax.net.ssl.X509ExtendedTrustManager;
>
> diff -r 3f9913affb06 tests/netx/unit/net/sourceforge/jnlp/cache/ResourceTrackerTest.java
> --- a/tests/netx/unit/net/sourceforge/jnlp/cache/ResourceTrackerTest.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/tests/netx/unit/net/sourceforge/jnlp/cache/ResourceTrackerTest.java Tue Apr 29 13:48:19 2014 +0200
> @@ -291,10 +291,10 @@
> versionedFileForServerWithoutHeader.createNewFile();
>
> ResourceTracker rt = new ResourceTracker();
> - Resource r1 = Resource.getResource(testServer.getUrl(fileForServerWithHeader.getName()), null, UpdatePolicy.NEVER);
> - Resource r2 = Resource.getResource(testServerWithBrokenHead.getUrl(fileForServerWithoutHeader.getName()), null, UpdatePolicy.NEVER);
> - Resource r3 = Resource.getResource(testServer.getUrl(versionedFileForServerWithHeader.getName()), new Version("1.0"), UpdatePolicy.NEVER);
> - Resource r4 = Resource.getResource(testServerWithBrokenHead.getUrl(versionedFileForServerWithoutHeader.getName()), new Version("1.0"), UpdatePolicy.NEVER);
> + Resource r1 = Resource.getResource(new CURL(testServer.getUrl(fileForServerWithHeader.getName())), null, UpdatePolicy.NEVER);
> + Resource r2 = Resource.getResource(new CURL(testServerWithBrokenHead.getUrl(fileForServerWithoutHeader.getName())), null, UpdatePolicy.NEVER);
> + Resource r3 = Resource.getResource(new CURL(testServer.getUrl(versionedFileForServerWithHeader.getName())), new Version("1.0"), UpdatePolicy.NEVER);
> + Resource r4 = Resource.getResource(new CURL(testServerWithBrokenHead.getUrl(versionedFileForServerWithoutHeader.getName())), new Version("1.0"), UpdatePolicy.NEVER);
> assertOnServerWithHeader(rt.findBestUrl(r1));
> assertVersionedOneOnServerWithHeader(rt.findBestUrl(r3));
> assertOnServerWithoutHeader(rt.findBestUrl(r2));
> @@ -330,26 +330,26 @@
>
> }
>
> - private void assertOnServerWithHeader(URL u) {
> - assertCommonComponentsOfUrl(u);
> - assertPort(u, testServer.getPort());
> + private void assertOnServerWithHeader(CURL u) {
> + assertCommonComponentsOfUrl(u.getUrl());
> + assertPort(u.getUrl(), testServer.getPort());
> }
>
> - private void assertVersionedOneOnServerWithHeader(URL u) {
> - assertCommonComponentsOfUrl(u);
> - assertPort(u, testServer.getPort());
> - assertVersion(u);
> + private void assertVersionedOneOnServerWithHeader(CURL u) {
> + assertCommonComponentsOfUrl(u.getUrl());
> + assertPort(u.getUrl(), testServer.getPort());
> + assertVersion(u.getUrl());
> }
>
> - private void assertOnServerWithoutHeader(URL u) {
> - assertCommonComponentsOfUrl(u);
> - assertPort(u, testServerWithBrokenHead.getPort());
> + private void assertOnServerWithoutHeader(CURL u) {
> + assertCommonComponentsOfUrl(u.getUrl());
> + assertPort(u.getUrl(), testServerWithBrokenHead.getPort());
> }
>
> - private void assertVersionedOneOnServerWithoutHeader(URL u) {
> - assertCommonComponentsOfUrl(u);
> - assertPort(u, testServerWithBrokenHead.getPort());
> - assertVersion(u);
> + private void assertVersionedOneOnServerWithoutHeader(CURL u) {
> + assertCommonComponentsOfUrl(u.getUrl());
> + assertPort(u.getUrl(), testServerWithBrokenHead.getPort());
> + assertVersion(u.getUrl());
> }
>
> private void assertCommonComponentsOfUrl(URL u) {
> diff -r 3f9913affb06 tests/netx/unit/net/sourceforge/jnlp/cache/ResourceUrlCreatorTest.java
> --- a/tests/netx/unit/net/sourceforge/jnlp/cache/ResourceUrlCreatorTest.java Tue Apr 15 11:32:02 2014 +0200
> +++ b/tests/netx/unit/net/sourceforge/jnlp/cache/ResourceUrlCreatorTest.java Tue Apr 29 13:48:19 2014 +0200
> @@ -22,12 +22,12 @@
>
> private URL getResultUrl(String url, Version version,
> boolean usePack /*use pack.gz suffix*/, boolean useVersion /*use version suffix*/) throws MalformedURLException {
> - Resource resource = Resource.getResource(new URL(url), version, null);
> + Resource resource = Resource.getResource(new CURL(new URL(url)), version, null);
> return ResourceUrlCreator.getUrl(resource, usePack, useVersion);
> }
>
> private URL getResultUrl(String url, Version version, DownloadOptions downloadOptions) throws MalformedURLException {
> - Resource resource = Resource.getResource(new URL(url), version, null);
> + Resource resource = Resource.getResource(new CURL(new URL(url)), version, null);
> ResourceUrlCreator ruc = new ResourceUrlCreator(resource, downloadOptions);
> return ruc.getVersionedUrl();
> }
More information about the distro-pkg-dev
mailing list