[RFC][Icedtea-Web]: Enforce cache limit based on deployment.properties
Dr Andrew John Hughes
ahughes at redhat.com
Mon Mar 21 11:51:46 PDT 2011
On 14:18 Mon 21 Mar , Andrew Su wrote:
>
snip..
>
> The changes to documentation and formatting was unintentional. I've fixed the patch to only have relavent changes.
>
Thanks. Do you have a ChangeLog for this? I'm not sure exactly what each of these changes are for.
snip...
> Cheers,
> Andrew
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/cache/CacheEntry.java
> --- a/netx/net/sourceforge/jnlp/cache/CacheEntry.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/cache/CacheEntry.java Mon Mar 21 14:19:43 2011 -0400
> @@ -162,4 +162,16 @@
> properties.store();
> }
>
> + /**
> + * Make this file appear in the temp cache folder. Once marked as temp, it
> + * will remain in temp until we have cleared the cache.
> + */
> + public void setTemp() {
> + File infoFile = CacheUtil.getCacheFile(location, version, "temp");
> + infoFile = new File(infoFile.getPath() + ".info"); // replace with something that can't be clobbered
> + properties = new PropertiesFile(infoFile, R("CAutoGen"));
> + properties.setProperty("isTemp", "true");
> + properties.store();
> + }
> +
> }
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/cache/CacheUtil.java
> --- a/netx/net/sourceforge/jnlp/cache/CacheUtil.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/cache/CacheUtil.java Mon Mar 21 14:19:43 2011 -0400
> @@ -21,6 +21,7 @@
> import java.io.*;
> import java.net.*;
> import java.nio.channels.FileChannel;
> +import java.nio.channels.FileLock;
> import java.util.*;
> import java.security.*;
> import javax.jnlp.*;
> @@ -29,6 +30,7 @@
> import net.sourceforge.jnlp.config.DeploymentConfiguration;
> import net.sourceforge.jnlp.runtime.*;
> import net.sourceforge.jnlp.util.FileUtils;
> +import net.sourceforge.jnlp.util.PropertiesFile;
>
> /**
> * Provides static methods to interact with the cache, download
> @@ -39,6 +41,9 @@
> */
> public class CacheUtil {
>
> + public static final String cacheDir = JNLPRuntime.getConfiguration().getProperty(DeploymentConfiguration.KEY_USER_CACHE_DIR);
> + public static final String cacheInfo = cacheDir + File.separator + "cache.info";
> +
> /**
> * Compares a URL using string compare of its protocol, host,
> * port, path, query, and anchor. This method avoids the host
> @@ -138,8 +143,7 @@
> return;
> }
>
> - File cacheDir = new File(JNLPRuntime.getConfiguration()
> - .getProperty(DeploymentConfiguration.KEY_USER_CACHE_DIR));
> + File cacheDir = new File(CacheUtil.cacheDir);
> if (!(cacheDir.isDirectory())) {
> return;
> }
> @@ -279,15 +283,33 @@
> * @throws IllegalArgumentException if the source is not cacheable
> */
> public static File getCacheFile(URL source, Version version) {
> + if (checkIfTemp(source, version)) {
> + return getCacheFile(source, version, "temp");
> + } else {
> + return getCacheFile(source, version, "perm");
> + }
> + }
> +
> + /**
> + * Returns the file for the locally cached contents of the
> + * source. This method returns the file location only and does
> + * not download the resource. The latest version of the
> + * resource that matches the specified version will be returned.
> + *
> + * @param source the source URL
> + * @param version the version id of the local file
> + * @param subdir subdirectory inside cache folder
> + * @return the file location in the cache, or null if no versions cached
> + * @throws IllegalArgumentException if the source is not cacheable
> + */
> + protected static File getCacheFile(URL source, Version version, String subdir) {
> // ensure that version is an version id not version string
>
> if (!isCacheable(source, version))
> throw new IllegalArgumentException(R("CNotCacheable", source));
>
> try {
> - String cacheDir = JNLPRuntime.getConfiguration()
> - .getProperty(DeploymentConfiguration.KEY_USER_CACHE_DIR);
> - File localFile = urlToPath(source, cacheDir);
> + File localFile = urlToPath(source, cacheDir + File.separator + subdir);
> FileUtils.createParentDir(localFile);
>
> return localFile;
> @@ -436,4 +458,173 @@
> }
> }
>
> + /**
> + * Get the size of directory by reading "cache.info" if it exists.
> + * Otherwise, call updateCacheDirSize() to create it.
> + *
> + * @return permanent cache directory size.
> + */
> + public synchronized static long getCacheDirSize() {
> + // This is an attempt to save directory traversal.
> +
> + File infoFile = new File(cacheInfo);
> + long size = Long.MAX_VALUE;
> +
> + FileLock fl = null;
> + try {
> + if (!infoFile.exists()) {
> + return updateCacheDirSize(); // This will be up to date.
> + }
> + fl = FileUtils.getFileLock(infoFile.getCanonicalPath(), true, true);
> + PropertiesFile properties = new PropertiesFile(infoFile);
> +
> + try {
> + size = Long.valueOf(properties.getProperty("curSize", String.valueOf(Long.MAX_VALUE)));
> + } catch (NumberFormatException e) { // Someone decided to be funny and added non-numeric values...
> + // We will fix this by calling update. However we need to release the lock first.
> + fl.release();
> + fl.channel().close();
> + return updateCacheDirSize();
> + }
> +
> + if (fl != null) {
> + fl.release();
> + fl.channel().close();
> + }
> + } catch (IOException e) {
> + }
> + return size;
> + }
> +
> + /**
> + * Get the permanent cache directory's size and write it to "cache.info".
> + *
> + * @return Size of the permanent cache directory.
> + * @throws IOException
> + */
> + public synchronized static long updateCacheDirSize() throws IOException {
> +
> + FileLock fl = null;
> + long size = Long.MAX_VALUE;
> +
> + try {
> + try {
> + File infoFile = new File(cacheInfo);
> + if (!infoFile.exists()) { // Let's try and make the file.
> + FileUtils.createParentDir(infoFile);
> + FileUtils.createRestrictedFile(infoFile, true); // Make it now so we can lock.
> + }
> + String canonicalPath = infoFile.getCanonicalPath();
> + fl = FileUtils.getFileLock(canonicalPath, false, true);
> + PropertiesFile properties = new PropertiesFile(infoFile, "Miscellaneous cache information.");
> + File permDir = new File(cacheDir + File.separator + "perm");
> + size = FileUtils.getDirSize(permDir, permDir); // We only want the size of permanent directory.
> + System.out.println(size);
> + properties.setProperty("curSize", String.valueOf(size));
> + properties.store();
> + } finally {
> + if (fl != null) {
> + fl.release();
> + fl.channel().close();
> + fl = null;
> + }
> + }
> + } catch (IOException e) {
> + e.printStackTrace();
> + }
> +
> + return size;
> + }
> +
> + /**
> + * Check if the file we are about to download will fit in the permanent
> + * cache directory. Reflect this future change to the cache info file.
> + *
> + * @param location the source URL.
> + * @param version the version id of the local file.
> + * @param size size of the file to be downloaded.
> + * @return true if it can fit in permanent cache directory, false otherwise.
> + */
> + public synchronized static boolean canFitPermCache(URL location, Version version, long size) {
> + String strMaxSize = JNLPRuntime.getConfiguration().getProperty("deployment.cache.max.size");
> + File localFile = getCacheFile(location, version, "perm");
> + long dirSize = getCacheDirSize();
> + if (localFile.isFile()) dirSize -= localFile.length();
> + dirSize = dirSize / (1024 * 1024);
> + // Allow only if unlimited or under threshold.
> + if (strMaxSize == "-1" || (size > -1 && size / (1024 * 1024) + dirSize < Long.valueOf(strMaxSize))) {
> + // We need to update the size in here before returning, since during the time after return to the
> + // time of download, another file may have already been downloaded.
> + changeCacheSize(location, version, size);
> +
> + return true;
> + }
> + return false;
> + }
> +
> + /**
> + * Removes the temporary cache directory only if there are no other
> + * instances of plugin or javaws running.
> + */
> + public static void clearTempCache() {
> + // We must be marked as stopped running before calling this.
> + // Otherwise we may get an exception since the lock may not be released yet.
> + // Adding this as its own shutdown hook doesn't work either, the hooks are run simultaneously.
> + if (okToClearCache()) {
> + File f = new File(cacheDir + File.separator + "temp");
> + if (!f.exists()) return;
> +
> + try {
> + FileUtils.recursiveDelete(f, f);
> + } catch (IOException e) {
> + e.printStackTrace();
> + }
> + }
> + }
> +
> + /**
> + * Check if we have cached it in the temporary directory.
> + *
> + * @param source the source URL
> + * @param version the version id of the local file.
> + * @return true if the info file exists, false otherwise.
> + */
> + public static boolean checkIfTemp(URL source, Version version) {
> + File f = urlToPath(source, cacheDir + File.separator + "temp");
> + f = new File(f.getPath() + ".info");
> + return f.isFile();
> + }
> +
> + /**
> + * Update the size of directory, it may increase or decrease.
> + *
> + * @param location the location of the file to be downloaded.
> + * @param version the version id of the local file.
> + * @param size length of the file to be downloaded.
> + */
> + public synchronized static void changeCacheSize(URL location, Version version, long size) {
> + FileLock fl = null;
> + try {
> + try {
> + fl = FileUtils.getFileLock(CacheUtil.cacheInfo, false, true); // Block until we can get an exclusive lock.
> + File localFile = getCacheFile(location, version, "perm");
> + PropertiesFile cacheInfo = new PropertiesFile(new File(CacheUtil.cacheInfo));
> + long localSize = Long.valueOf(cacheInfo.getProperty("curSize", String.valueOf(Long.MAX_VALUE)));
> + if (localFile.exists()) localSize -= localFile.length();
> + if (localSize <= Long.MAX_VALUE - size) { // Prevent overflow.
> + localSize += size; // This is fine since new file might be smaller.
> + cacheInfo.setProperty("curSize", String.valueOf(localSize));
> + cacheInfo.store();
> + }
> + } finally {
> + if (fl != null) {
> + fl.release();
> + fl.channel().close();
> + }
> + }
> + } catch (IOException e) {
> + e.printStackTrace();
> + }
> + }
> +
> }
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/cache/ResourceTracker.java
> --- a/netx/net/sourceforge/jnlp/cache/ResourceTracker.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/cache/ResourceTracker.java Mon Mar 21 14:19:43 2011 -0400
> @@ -693,8 +693,11 @@
> CacheUtil.getCacheFile(downloadLocation, resource.downloadVersion)));
> InputStream inputStream = new BufferedInputStream(gzInputStream);
>
> - JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(
> - CacheUtil.getCacheFile(resource.location, resource.downloadVersion)));
> + // We want to keep this in temp, since we don't know the size, and moving is expensive.
> + CacheEntry output = new CacheEntry(resource.location, resource.downloadVersion);
> + output.setTemp();
> + resource.localFile = CacheUtil.getCacheFile(resource.location, resource.downloadVersion);
> + JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(resource.localFile));
>
> Unpacker unpacker = Pack200.newUnpacker();
> unpacker.unpack(inputStream, outputStream);
> @@ -707,9 +710,11 @@
> .getCacheFile(downloadLocation, resource.downloadVersion)));
> InputStream inputStream = new BufferedInputStream(gzInputStream);
>
> - BufferedOutputStream outputStream = new BufferedOutputStream(
> - new FileOutputStream(CacheUtil.getCacheFile(resource.location,
> - resource.downloadVersion)));
> + // We want to keep this in temp, since we don't know the size, and moving is expensive.
> + CacheEntry output = new CacheEntry(resource.location, resource.downloadVersion);
> + output.setTemp();
> + resource.localFile = CacheUtil.getCacheFile(resource.location, resource.downloadVersion);
> + BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(resource.localFile));
>
> while (-1 != (rlen = inputStream.read(buf))) {
> outputStream.write(buf, 0, rlen);
> @@ -755,6 +760,12 @@
>
> int size = connection.getContentLength();
> boolean current = CacheUtil.isCurrent(resource.location, resource.requestVersion, connection) && resource.getUpdatePolicy() != UpdatePolicy.FORCE;
> + // If it is current, we wouldn't need to download anyways. So no change. If not current...
> + CacheEntry entry = new CacheEntry(resource.location, resource.requestVersion);
> + if (!current && !CacheUtil.canFitPermCache(resource.location, resource.requestVersion, size)){
> + entry.setTemp(); // Can not be placed back into perm unless we start over.
> + localFile = CacheUtil.getCacheFile(resource.location, resource.downloadVersion); // Get the new location.
> + }
>
> synchronized (resource) {
> resource.localFile = localFile;
> @@ -768,7 +779,6 @@
> }
>
> // update cache entry
> - CacheEntry entry = new CacheEntry(resource.location, resource.requestVersion);
> if (!current)
> entry.initialize(connection);
>
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/runtime/JNLPClassLoader.java
> --- a/netx/net/sourceforge/jnlp/runtime/JNLPClassLoader.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/runtime/JNLPClassLoader.java Mon Mar 21 14:19:43 2011 -0400
> @@ -166,11 +166,11 @@
> // initialize extensions
> initializeExtensions();
>
> + initializeResources();
> +
> // initialize permissions
> initializePermissions();
>
> - initializeResources();
> -
> setSecurity();
>
> installShutdownHooks();
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/runtime/JNLPRuntime.java
> --- a/netx/net/sourceforge/jnlp/runtime/JNLPRuntime.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/runtime/JNLPRuntime.java Mon Mar 21 14:19:43 2011 -0400
> @@ -19,7 +19,6 @@
> import java.io.*;
> import java.net.Authenticator;
> import java.net.ProxySelector;
> -import java.nio.channels.FileChannel;
> import java.nio.channels.FileLock;
> import java.awt.*;
> import java.text.*;
> @@ -653,25 +652,9 @@
> fos.close();
> }
> }
> -
> - FileInputStream is = new FileInputStream(netxRunningFile);
> - FileChannel channel = is.getChannel();
> - fileLock = channel.lock(0, 1, true);
> - if (!fileLock.isShared()){ // We know shared locks aren't offered on this system.
> - FileLock temp = null;
> - for (long pos = 1; temp == null && pos < Long.MAX_VALUE - 1; pos++){
> - temp = channel.tryLock(pos, 1, false); // No point in requesting for shared lock.
> - }
> - fileLock.release(); // We can release now, since we hold another lock.
> - fileLock = temp; // Keep the new lock so we can release later.
> - }
>
> - if (fileLock != null && fileLock.isShared()) {
> - if (JNLPRuntime.isDebug()) {
> - System.out.println("Acquired shared lock on " +
> - netxRunningFile.toString() + " to indicate javaws is running");
> - }
> - }
> + fileLock = FileUtils.getFileLock(netxRunningFile.getCanonicalPath(), true, true);
> +
> } catch (IOException e) {
> e.printStackTrace();
> }
> @@ -679,6 +662,7 @@
> Runtime.getRuntime().addShutdownHook(new Thread() {
> public void run() {
> markNetxStopped();
> + CacheUtil.clearTempCache();
> }
> });
> }
> diff -r a640b4e4d226 netx/net/sourceforge/jnlp/util/FileUtils.java
> --- a/netx/net/sourceforge/jnlp/util/FileUtils.java Mon Mar 21 17:27:20 2011 +0100
> +++ b/netx/net/sourceforge/jnlp/util/FileUtils.java Mon Mar 21 14:19:43 2011 -0400
> @@ -19,7 +19,12 @@
> import static net.sourceforge.jnlp.runtime.Translator.R;
>
> import java.io.File;
> +import java.io.FileNotFoundException;
> import java.io.IOException;
> +import java.io.RandomAccessFile;
> +import java.nio.channels.FileChannel;
> +import java.nio.channels.FileLock;
> +import java.util.HashMap;
>
> import net.sourceforge.jnlp.runtime.JNLPRuntime;
>
> @@ -292,4 +297,84 @@
>
> }
>
> + /**
> + * Get the size of the given directory. All symlinks are ignored. Hardlinks
> + * will be counted.
> + *
> + * @param file The file we are checking.
> + * @param baseDir The current directory we are in.
> + * @return size of the given directory.
> + * @throws IOException on an IO exception.
> + */
> + public static long getDirSize(File file, File baseDir) throws IOException {
> + long size = 0;
> +
> + String fileCanon = file.getCanonicalPath();
> + String baseCanon = baseDir.getCanonicalPath();
> + // This is a symlink to a directory such that the current directory is not the parent of.
> + if (!(fileCanon.startsWith(baseCanon))) return 0;
> +
> + // This is a symlink to a file in a subdirectory.
> + if (!(fileCanon.equals(baseCanon)) && !(file.getCanonicalFile().getParent().equals(baseCanon))) return 0;
> +
> + // We still need to take care of the case where it is a link to a file in the current directory.
> + // We will do that check by keeping a copy of the files in the current directory that we checked.
> + // That way even if we encounter the symbolic link, we can skip it. In the other case where we
> + // encounter the symbolic link first we can ignore the check on original, since results are the same.
> +
> + HashMap<String, Object> map = new HashMap<String, Object>();
> + if (file.isDirectory()) {
> + for (File child : file.listFiles()){
> + fileCanon = child.getCanonicalPath(); // Reuse variable
> + if (!map.containsKey(fileCanon)) {
> + size += getDirSize(child, file);
> + map.put(fileCanon, null);
> + }
> + }
> + }
> + if (file.isFile()) {
> + size = file.length();
> + }
> +
> + return size;
> + }
> +
> + /**
> + * This will return a lock to the file specified.
> + *
> + * @param name Name of the file to get lock from.
> + * @param shared Specify if the lock will be a shared lock.
> + * @param allowBlock Specify if we should block when we can not get the
> + * lock. No effect when trying to get shared lock.
> + * @return FileLock if we were successful in getting a lock, otherwise null.
> + * @throws FileNotFoundException If the file does not exist.
> + */
> + public static FileLock getFileLock(String name, boolean shared, boolean allowBlock) throws FileNotFoundException{
> + RandomAccessFile rafFile = new RandomAccessFile(name, "rw");
> + FileChannel fc = rafFile.getChannel();
> + FileLock lock = null;
> + try {
> + if (!shared) {
> + if (allowBlock) {
> + lock = fc.lock(0, Long.MAX_VALUE, false);
> + } else {
> + lock = fc.tryLock(0, Long.MAX_VALUE, false);
> + }
> + } else { // We want shared lock. This will block regardless if allowBlock is true or not.
> + // Test to see if we can get a shared lock.
> + lock = fc.lock(0, 1, true); // Block if a non exclusive lock is being held.
> + if (!lock.isShared()) { // This lock is an exclusive lock. Use alternate solution.
> + FileLock tempLock = null;
> + for (long pos = 1; tempLock == null && pos < Long.MAX_VALUE - 1; pos++) {
> + tempLock = fc.tryLock(pos, 1, false);
> + }
> + lock.release();
> + lock = tempLock; // Get the unique exclusive lock.
> + }
> + }
> + } catch (IOException e) {
> + e.printStackTrace();
> + }
> + return lock;
> + }
> }
--
Andrew :)
Free Java Software Engineer
Red Hat, Inc. (http://www.redhat.com)
Support Free Java!
Contribute to GNU Classpath and IcedTea
http://www.gnu.org/software/classpath
http://icedtea.classpath.org
PGP Key: F5862A37 (https://keys.indymedia.org/)
Fingerprint = EA30 D855 D50F 90CD F54D 0698 0713 C3ED F586 2A37
More information about the distro-pkg-dev
mailing list