ReflectiveEnvironment#isPackage() refactored to provide better performance.
In a nutshell, when a .jar file is opened a full list of package is built
so each .jar is opened and scanned only once. Also, this list ot package is
cached (as long as .jar is not modified) to avoid multiple scanning among
multiple compilation cycles. This makes interactive tools much more responsive
as scanning .jar was one of the bottlenecks.
--- a/tools/java/src-tests/stx/libjava/tools/compiler/tests/CompilerAdapterTests.java Fri Aug 08 11:02:10 2014 +0100
+++ b/tools/java/src-tests/stx/libjava/tools/compiler/tests/CompilerAdapterTests.java Mon Aug 11 10:55:57 2014 +0100
@@ -287,6 +287,21 @@
assertEquals(RetentionPolicy.RUNTIME, retention.value());
}
+
+ @Test
+ public void test_10() {
+ ClassLoader l = new ClassLoader();
+ CompilerAdapter c = new CompilerAdapter(l);
+
+ c.compile("public class Foo { public void f() { } }");
+ assertEquals(1, c.getClassFiles().length);
+ assertFalse(c.getResult().hasErrors());
+ l.load(c.getClassFiles()[0].getBytes());
+
+ c.compile("public class Bar { public void f() { new Foo(); } }");
+ assertEquals(1, c.getClassFiles().length);
+ assertFalse(c.getResult().hasErrors());
+ }
}
--- a/tools/java/src/stx/libjava/tools/environment/ReflectiveEnvironment.java Fri Aug 08 11:02:10 2014 +0100
+++ b/tools/java/src/stx/libjava/tools/environment/ReflectiveEnvironment.java Mon Aug 11 10:55:57 2014 +0100
@@ -12,10 +12,7 @@
import java.util.Iterator;
import java.util.Map;
import java.util.Vector;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import org.eclipse.jdt.internal.compiler.classfmt.ClassFileReader;
@@ -23,17 +20,25 @@
import org.eclipse.jdt.internal.compiler.env.IBinaryType;
import org.eclipse.jdt.internal.compiler.env.INameEnvironment;
import org.eclipse.jdt.internal.compiler.env.NameEnvironmentAnswer;
+//import java.util.logging.Level;
+//import java.util.logging.Logger;
+
+import stx.libjava.tools.environment.internal.ReflectiveClassLoader;
public class ReflectiveEnvironment implements INameEnvironment {
- private static final Logger LOGGER = Logger.getLogger(ReflectiveEnvironment.class.getName());
+ /*
+ * No, do not log for now. The problem is that on (Open)JDK 7 this trigger
+ * some not-yet-implemented natives in java.nio.*. Sad...
+ */
+ //private static final Logger LOGGER = Logger.getLogger(ReflectiveEnvironment.class.getName());
private static final boolean DEBUG_isPackage = false;
private static Field FIELD_ClassLoader_Classes;
/**
- * A java.lang.ClassLoader where to search for classes & packages
+ * A ReflectiveClassLoader where to search for classes & packages
*/
- protected ClassLoader classloader;
+ protected ReflectiveClassLoader classloader;
/**
* Internal cache of found/not found packages to speed-up subsequent calls to
@@ -69,11 +74,11 @@
public ReflectiveEnvironment(TypeRegistry types, ClassLoader cl) {
this.types = types;
- this.classloader = cl;
+ this.classloader = ReflectiveClassLoader.forClassLoader(cl);
}
public ClassLoader getClassLoader() {
- return classloader;
+ return classloader.getRealClassLoader();
}
/**
@@ -107,7 +112,7 @@
if (type != null) return type;
try {
- Class<?> c = classloader.loadClass(name);
+ Class<?> c = getClassLoader().loadClass(name);
IBinaryType ctype = new ReflectiveType(c);
types.put(name, ctype);
return ctype;
@@ -152,194 +157,32 @@
* isPackage(null, {java});
*/
- public boolean isPackage(char[][] parentPackageName, char[] packageName) {
- return isPackage(concat(parentPackageName, packageName));
- }
-
- /**
- * Answer whether packageName is the name of a known package
- * The default package is always assumed to exist.
- *
- * For example:
- * isPackage("java.awt.event");
- * isPackage("java");
- */
- public boolean isPackage(String packageName) {
+ public boolean isPackage(char[][] parentPackageName, char[] subPackageName) {
/* First, consult cached value to speed up second and later calls
- * with the same packageName. The cache is per-instance and thus valid
- * only during the compilation.
- * In theory it may happen that cached value is no longer valid due to
- * changes to classloader paths <em>during</em> single compilation cycle,
- * but we consider this as a trade-off for speed. It seems that compiler itself
- * does not bother to cache results and keeps asking for the same package names
- * over and over.
- */
- Boolean isPackageB = cachedIsPackageResults.get(packageName);
- if (isPackageB != null) {
- return isPackageB.booleanValue();
- }
- boolean isPackage;
- if (Package.getPackage(packageName) != null) {
- isPackage = true;
- } else {
- isPackage = isPackage(packageName, classloader);
- }
- cachedIsPackageResults.put(packageName, isPackage);
- return isPackage;
- }
-
- /**
- * Answer whether @argument packageName is the name of a known package
- * within class loader @argument loader.
- * The default package is always assumed to exist.
- *
- */
- @SuppressWarnings("unchecked")
- public boolean isPackage(String packageName, ClassLoader loader) {
- // Primordial class loader, consult boot class path.
- if (loader == null) {
- return isPackage(packageName, sun.misc.Launcher.getBootstrapClassPath().getURLs());
- }
- // Ask parent class loader...
- if (isPackage(packageName, loader.getParent())) {
- return true;
- }
-
- /*
- * Another performance optimization: consult the list of classes already loaded by
- * given classloader. That list is not readable by standard ClassLoader API though this
- * hack. On OpenJDK-based environments, there's a field {@link ClassLoader#classes}.
- * However, we may or may not have an access to it. If not, just continue and do slow
- * search through .jars and .class directories.
- */
- if (FIELD_ClassLoader_Classes != null) {
- int packageNameLen = packageName.length();
- Vector<Class<?>> classes;
- try {
- classes = (Vector<Class<?>>)FIELD_ClassLoader_Classes.get(loader);
- if (classes.size() > 0) {
- Iterator<Class<?>> i = classes.iterator();
- while (i.hasNext()) {
- String className = i.next().getName();
- int classNameLen = className.length();
- if ((classNameLen > packageNameLen) &&
- (className.charAt(packageNameLen) == '.') &&
- (className.startsWith(packageName))) {
- return true;
- } else if ((classNameLen == packageNameLen) &&
- className.equals(packageName)) {
- return false;
- }
- }
- }
- } catch (IllegalArgumentException e) {
- } catch (IllegalAccessException e) {
- }
- }
-
-
- if (loader instanceof URLClassLoader) {
- if (isPackage(packageName, ((URLClassLoader)loader).getURLs())) {
- return true;
- }
- }
- return false;
+ * with the same packageName. The cache is per-instance and thus valid
+ * only during the compilation.
+ * In theory it may happen that cached value is no longer valid due to
+ * changes to classloader paths <em>during</em> single compilation cycle,
+ * but we consider this as a trade-off for speed. It seems that compiler itself
+ * does not bother to cache results and keeps asking for the same package names
+ * over and over.
+ */
+ String packageName = concat(parentPackageName, subPackageName);
+ Boolean isPackageB = cachedIsPackageResults.get(packageName);
+ if (isPackageB != null) {
+ return isPackageB.booleanValue();
+ }
+ boolean isPackage;
+ if (Package.getPackage(packageName) != null) {
+ isPackage = true;
+ } else {
+ isPackage = classloader.isPackage(packageName);
+ }
+ cachedIsPackageResults.put(packageName, isPackage);
+ return isPackage;
}
/**
- * Answer whether @argument packageName is the name of a known package
- * along passed @argument classpath.
- * The default package is always assumed to exist.
- */
- public boolean isPackage(String packageName, URL[] classpath) {
- int packageNameLen = packageName.length();
- String packagePathSlashed = packageName.replace('.', '/');
- String packagePathNative;
- if (File.separatorChar == '/') {
- packagePathNative = packagePathSlashed;
- } else {
- packagePathNative = packageName.replace('.', File.separatorChar);
- }
- ZipFile[] zips = new ZipFile[classpath.length];
-
- try {
- // Pass 1 - quick check for directories and zip files
- for (int i = 0; i < classpath.length; i++) {
- URL url = classpath[i];
- if (url.getProtocol().equals("file")) {
- File file = new File(url.getPath());
- if (file.isDirectory()) {
- if ((new File(file, packagePathNative)).isDirectory()) {
- return true;
- }
- } else if (! file.canExecute()){
- /* Sometimes it happen what we get *.so in class path. Ignore it, it won't
- * open as .zip / .jar files.
- */
- ZipFile zip = null;
- try {
-
- zip = new ZipFile(file);
- if (DEBUG_isPackage) System.err.println("checking jar: " + file.getName());
- /*
- * Try to short-circuit: look for .class file assuming
- * packageName is actually class name. If found, return
- * false (it is a class, not a package)
- */
- if (zip.getEntry(packagePathSlashed + ".class") != null) {
- return false;
- } else {
- zips[i] = zip;
- }
- } catch (IOException e) {
- /*
- * No, do not log for now. The problem is that on (Open)JDK 7 this trigger
- * some not-yet-implemented natives in java.nio.*. Sad...
- */
- // LOGGER.log(Level.INFO, "Failed to open .jar file: " + url.getPath() , e);
- System.err.println("Failed to open .jar file: " + url.getPath());
- }
- }
- }
- }
-
- // Pass 2 - full scan of zip files
- for (int i = 0; i < zips.length; i++) {
- ZipFile zip = zips[i];
- if (zip != null) {
- Enumeration<? extends ZipEntry> entries = zip.entries();
- while (entries.hasMoreElements()) {
- String entryName = entries.nextElement().getName();
- int entryNameLen = entryName.length();
- if (DEBUG_isPackage) {
- System.err.println("checking jar entry: " + entryName);
- }
-
- if ((entryNameLen > packageNameLen) &&
- (entryName.charAt(packageNameLen) == '/') &&
- (entryName.startsWith(packagePathSlashed))) {
- return true;
- }
- }
- }
- }
- } finally {
- for (int i = 0; i < zips.length; i++) {
- ZipFile zip = zips[i];
- if (zip != null) {
- try {
- zip.close();
- } catch (IOException e) {
- }
- }
- }
- }
-
- return false;
- }
-
-
- /**
* This method cleans the environment. It is responsible for releasing the memory
* and freeing resources. Passed that point, the name environment is no longer usable.
*
@@ -406,9 +249,28 @@
}
}
if (name != null) {
- if (pkg != null) sb.append('.');
+ if (pkg != null && pkg.length != 0) {
+ if (pkg != null) sb.append('.');
+ }
sb.append(name);
}
return sb.toString();
}
+
+ public static abstract class Library {
+ protected File file;
+
+
+
+ public abstract boolean isPackage(String packageName);
+
+ public static class Directory extends Library {
+
+ @Override
+ public boolean isPackage(String packageName) {
+ return false;
+ }
+
+ }
+ }
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/java/src/stx/libjava/tools/environment/internal/ReflectiveClassLoader.java Mon Aug 11 10:55:57 2014 +0100
@@ -0,0 +1,159 @@
+/**
+ *
+ */
+package stx.libjava.tools.environment.internal;
+
+import java.lang.reflect.Field;
+import java.util.Iterator;
+import java.util.Vector;
+
+
+/**
+ * <code>ReflectiveClassLoader</code> is a simple wrapper on a {@link java.lang.ClassLoader} that provides
+ * reflective features standard {@link java.lang.ClassLoader} doesn't.
+ * In particular, it provides {@link #isPackage(String)} which is used
+ * by {@link stx.libjava.tools.environment.ReflectiveEnvironment}.
+ *
+ * The way {@link java.lang.ClassLoader} accesses and loads classes differ
+ * so specialised subclasses exists for individual {@link java.lang.ClassLoader}
+ * instances in order to provide more accurate answers. Therefore use method
+ * {@link #forClassLoader(ClassLoader)} to create an instance of a
+ * <code>ReflectiveClassLoader</code>
+ *
+ * @author Jan Vrany <jan.vrany [at] fit.cvut.cz>
+ *
+ */
+@stx.libjava.annotation.Package("stx:libjava/tools")
+public class ReflectiveClassLoader {
+ /**
+ * Shared cached {@link ReflectiveClassLoader} representing primordial class loader.
+ * Primordial class loader is not to change during the life-span of a JVM, so it can be
+ * safely cached.
+ *
+ */
+ protected static ReflectiveClassLoader PRIMORDIAL = new ReflectiveURLClassLoader(null, sun.misc.Launcher.getBootstrapClassPath().getURLs());
+
+ /**
+ * Cached {@link Field} to access normally inaccessible field {@link java.lang.ClassLoader#classes}.
+ * On OpenJDK based libraries, this field contains a list of loaded classes. This list is used
+ * to check for loaded classes as a short-circuit (or as the only way for generic class loaders)
+ */
+ protected static Field FIELD_ClassLoader_Classes;
+
+ /**
+ * A real {@link ClassLoader} this {@link ReflectiveClassLoader} reflects on.
+ * For primordial class loader the value is <code>null</code>.
+ */
+ protected ClassLoader realClassLoader;
+
+ /**
+ * A cached {@link ReflectiveClassLoader} representing a parent class loader of
+ * this object. For primordial class loader the value is <code>null</code>.
+ */
+ protected ReflectiveClassLoader parent;
+
+ protected ReflectiveClassLoader(ClassLoader loader) {
+ realClassLoader = loader;
+ }
+
+ static {
+ try {
+ FIELD_ClassLoader_Classes = ClassLoader.class.getDeclaredField("classes");
+ FIELD_ClassLoader_Classes.setAccessible(true);
+ } catch (NoSuchFieldException e) {
+ FIELD_ClassLoader_Classes = null;
+ } catch (SecurityException e) {
+ FIELD_ClassLoader_Classes = null;
+ }
+ }
+
+ /**
+ * Returns a suitable {@link ReflectiveClassLoader} for given (real) {@link ClassLoader}.
+ *
+ * @param loader a {@link ClassLoader} on which to reflect.
+ * @return a {@link ReflectiveClassLoader} for given @param loader.
+ */
+ public static ReflectiveClassLoader forClassLoader(ClassLoader loader) {
+ if (loader == null) {
+ return PRIMORDIAL;
+ }
+ /* This is kind of ugly, we should provide a way to register custom subclasses,
+ * but for now, keep it simple. We'll see if this machinery proves usable at all.
+ */
+ if (loader instanceof java.net.URLClassLoader) {
+ return new ReflectiveURLClassLoader((java.net.URLClassLoader)loader);
+ }
+ return new ReflectiveClassLoader(loader);
+ }
+
+ public ReflectiveClassLoader getParent() {
+ if (realClassLoader == null) return null;
+ if (parent == null) {
+ parent = ReflectiveClassLoader.forClassLoader(realClassLoader.getParent());
+ }
+ return parent;
+ }
+
+ /**
+ * Return <code>true</code> is package with name @param packageName exists under
+ * this class loader, <code>false</code> otherwise.
+ *
+ * A package is considered to exist if there's at least one class in that package.
+ * However, individual {@link ReflectiveClassLoader}s may extend the definition of
+ * {@link #isPackage(String)} For example {@link ReflectiveURLClassLoader} may return
+ * <code>true</code> if a directory exists along it's class path.
+ *
+ * @param packageName a name of the package
+ * @return <code>true</code> if such package exists, <code>false</code> otherwise
+ */
+ public boolean isPackage(String packageName) {
+ // First, check parent class loader
+ if (realClassLoader != null) {
+ if (realClassLoader.getParent() == null) {
+ if (PRIMORDIAL.isPackage(packageName)) {
+ return true;
+ }
+ } else {
+ if (getParent().isPackage(packageName)) {
+ return true;
+ }
+ }
+ }
+ // Second, consult list of already loaded classes.
+ if (realClassLoader != null && FIELD_ClassLoader_Classes != null) {
+ int packageNameLen = packageName.length();
+ Vector<Class<?>> classes;
+ try {
+ classes = (Vector<Class<?>>)FIELD_ClassLoader_Classes.get(realClassLoader);
+ if (classes.size() > 0) {
+ Iterator<Class<?>> i = classes.iterator();
+ while (i.hasNext()) {
+ String className = i.next().getName();
+ int classNameLen = className.length();
+ if ((classNameLen > packageNameLen) &&
+ (className.charAt(packageNameLen) == '.') &&
+ (className.startsWith(packageName))) {
+ return true;
+ } else if ((classNameLen == packageNameLen) &&
+ className.equals(packageName)) {
+ return false;
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ // If we're for whatever reason fobidden access to classes field,
+ // well, just ignore it.
+ } catch (IllegalAccessException e) {
+ // same as above.
+ }
+ }
+ // Last, we don't know. For generic ClassLoader, there's no other way
+ // to figure out what packages are there or not. So be conservative...
+ return false;
+ }
+
+ public ClassLoader getRealClassLoader() {
+ return realClassLoader;
+ }
+
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/java/src/stx/libjava/tools/environment/internal/ReflectiveURLClassLoader.java Mon Aug 11 10:55:57 2014 +0100
@@ -0,0 +1,166 @@
+package stx.libjava.tools.environment.internal;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/**
+ * A specialized {@link ReflectiveClassLoader} to reflect upon {@link URLClassLoader}.
+ * In addition to checks done in {@link ReflectiveClassLoader}, it looks along the class
+ * path for classes and packages available in order to provide more accurate answers.
+ *
+ * @author Jan Vrany <jan.vrany [at] fit.cvut.cz>
+ *
+ */
+public class ReflectiveURLClassLoader extends ReflectiveClassLoader {
+ protected URL[] classpath;
+ protected URL[] pending;
+ protected int pendingStart;
+ protected Library[] libraries;
+ protected int librariesCount;
+
+ /**
+ * A shared cache of already created libraries. Some libraries are huge
+ * so it pays off to cache them. Especially for .jar files which are
+ * costly to open and scan (specially under stx:libjava!)
+ */
+ protected static Map<File, Library> LIBRARIES = new HashMap<File, Library>();
+
+
+ protected ReflectiveURLClassLoader(URLClassLoader loader) {
+ this(loader, loader.getURLs());
+ }
+
+ protected ReflectiveURLClassLoader(URLClassLoader loader, URL[] classpath) {
+ super(loader);
+ this.classpath = classpath;
+ this.pending = new URL[classpath.length];
+ System.arraycopy(classpath, 0, pending, 0, classpath.length);
+ pendingStart = 0;
+ libraries = new ReflectiveURLClassLoader.Library[classpath.length];
+ librariesCount = 0;
+ }
+
+ public boolean isPackage(String packageName) {
+ // Try parent class loader, consult classes field....
+ if (super.isPackage(packageName)) {
+ return true;
+ }
+
+ // Look along already opened and scanned libraries...
+ for (int i = 0; i < librariesCount; i++) {
+ Library l = libraries[i];
+ if (l != null && l.isPackage(packageName)) {
+ return true;
+ }
+ }
+ // OK, no library already opened contains the package,
+ // so open more if there is more...
+ while (pendingStart < pending.length) {
+ File f = new File(pending[pendingStart].getPath());
+ Library l = LIBRARIES.get(f);
+ if (l == null && f.exists()) {
+ if (f.isDirectory()) {
+ l = new Dir(f);
+ LIBRARIES.put(f, l);
+ } else {
+ if (! f.getPath().endsWith("so") && ! f.getPath().endsWith("dll") ) {
+ l = new Jar(f);
+ LIBRARIES.put(f, l);
+ }
+ }
+ }
+ libraries[librariesCount] = l;
+ pendingStart++;
+ librariesCount++;
+ if (l != null && l.isPackage(packageName)) {
+ return true;
+ }
+ }
+ // Package not found...
+ return false;
+ }
+
+ protected static abstract class Library {
+ public abstract boolean isPackage(String packageName);
+ }
+
+ protected static class Jar extends ReflectiveURLClassLoader.Library {
+ protected File file;
+ protected long fileLastModified;
+ protected Set<String> packages;
+
+ protected Jar(File file) {
+ this.file = file;
+ }
+
+ public boolean isPackage(String packageName) {
+ if (packages == null || file.lastModified() > fileLastModified ) {
+ scan();
+ }
+ boolean isPackage = packages.contains(packageName);
+ return isPackage;
+ }
+
+ protected void scan() {
+ ZipFile zip = null;
+ packages = new HashSet<String>();
+ try {
+ zip = new ZipFile(file);
+ Enumeration<? extends ZipEntry> entries = zip.entries();
+ while (entries.hasMoreElements()) {
+ String entryName = entries.nextElement().getName();
+ if (entryName.endsWith(".class")) {
+ int slash = entryName.indexOf('/');
+ while (slash != -1) {
+ String packageName = entryName.substring(0, slash);
+ packages.add(packageName);
+ slash = entryName.indexOf('/', slash + 1);
+ }
+ }
+ }
+ } catch (IOException e) {
+ /*
+ * No, do not log for now. The problem is that on (Open)JDK 7 this trigger
+ * some not-yet-implemented natives in java.nio.*. Sad...
+ */
+ // LOGGER.log(Level.INFO, "Failed to open .jar file: " + url.getPath() , e);
+ System.err.println("Failed to open .jar file: " + file.getPath());
+ } finally {
+ if (zip != null) {
+ try {
+ zip.close();
+ } catch (IOException e) {
+ /*
+ * No, do not log for now. The problem is that on (Open)JDK 7 this trigger
+ * some not-yet-implemented natives in java.nio.*. Sad...
+ */
+ // LOGGER.log(Level.INFO, "Failed to open .jar file: " + url.getPath() , e);
+ System.err.println("Failed to close .jar file: " + file.getPath()); }
+ }
+ fileLastModified = file.lastModified();
+ }
+ }
+ }
+
+ protected static class Dir extends ReflectiveURLClassLoader.Library {
+ protected File directory;
+
+ protected Dir(File file) {
+ directory = file;
+ }
+
+ public boolean isPackage(String packageName) {
+ String packagePath = packageName.replace('.', File.separatorChar);
+ return new File(directory, packagePath).isDirectory();
+ }
+ }
+}
\ No newline at end of file