Skip to content

Commit 5e1631f

Browse files
sblivensbliven
committed
Improving memory efficiency of EcodFactory
Switch to a soft cache to avoid OutOfMemoryErrors.
1 parent ae8ca43 commit 5e1631f

File tree

2 files changed

+49
-8
lines changed

2 files changed

+49
-8
lines changed

‎biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/domain/EcodInstallationTest.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import org.biojava.nbio.structure.ecod.EcodDomain;
4444
import org.biojava.nbio.structure.ecod.EcodFactory;
4545
import org.biojava.nbio.structure.ecod.EcodInstallation;
46-
import org.junit.Ignore;
4746
import org.junit.Rule;
4847
import org.junit.Test;
4948
import org.junit.rules.TemporaryFolder;
@@ -233,7 +232,7 @@ public void testVersion() throws IOException {
233232
* Hierarchical field warnings are expected for versions prior to develop68.
234233
* @throws IOException
235234
*/
236-
//@Ignore
235+
//@Ignore // Very slow parsing test
237236
@Test
238237
public void testAllVersions() throws IOException {
239238
// Fetch latest version
@@ -243,6 +242,7 @@ public void testAllVersions() throws IOException {
243242
Matcher match = Pattern.compile("develop([0-9]+)",Pattern.CASE_INSENSITIVE).matcher(latestVersionStr);
244243
if(match.matches())
245244
latestVersion = Integer.parseInt(match.group(1));
245+
latest = null;
246246

247247
// List all versions
248248
int firstVersion = 45;
@@ -258,6 +258,20 @@ public void testAllVersions() throws IOException {
258258
EcodInstallation ecod = (EcodInstallation)EcodFactory.getEcodDatabase(version);
259259
ecod.getAllDomains();
260260
System.out.println(version +" -> "+ ecod.getVersion());
261+
262+
// Force garbage collection of all soft references
263+
// This shouldn't be required, but without it we get
264+
// 'OutOfMemoryError: GC overhead limit exceeded'.
265+
// Probably this is due to synchronization in EcodFactory blocking
266+
// the GC during parsing. -Spencer
267+
ecod = null;
268+
System.gc();
269+
try {
270+
@SuppressWarnings("unused")
271+
Object[] ignored = new Object[(int) Runtime.getRuntime().maxMemory()];
272+
} catch (Throwable e) {
273+
// Ignore OME
274+
}
261275
}
262276
}
263277
}

‎biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodFactory.java‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,12 @@
2121
package org.biojava.nbio.structure.ecod;
2222

2323
import java.io.IOException;
24+
import java.lang.ref.SoftReference;
2425
import java.util.Collections;
2526
import java.util.HashMap;
27+
import java.util.Iterator;
2628
import java.util.Map;
29+
import java.util.Map.Entry;
2730

2831
import org.biojava.nbio.structure.align.util.UserConfiguration;
2932
import org.biojava.nbio.structure.cath.CathDatabase;
@@ -46,8 +49,8 @@ public class EcodFactory {
4649

4750
public static String DEFAULT_VERSION = EcodInstallation.DEFAULT_VERSION;
4851

49-
private static Map<String, EcodDatabase> versionedEcodDBs =
50-
Collections.synchronizedMap(new HashMap<String, EcodDatabase>());
52+
private static Map<String, SoftReference<EcodDatabase>> versionedEcodDBs =
53+
Collections.synchronizedMap(new HashMap<String, SoftReference<EcodDatabase>>());
5154
private static String defaultVersion = DEFAULT_VERSION;
5255

5356
/**
@@ -61,27 +64,51 @@ public static EcodDatabase getEcodDatabase(String version) {
6164
if( version == null )
6265
version = defaultVersion;
6366

67+
logger.trace("Waiting for EcodFactory lock to get version "+version);
6468
synchronized(versionedEcodDBs) {
65-
EcodDatabase ecod = versionedEcodDBs.get(version.toLowerCase());
69+
logger.trace("Got EcodFactory lock to get version "+version);
70+
71+
releaseReferences();
72+
73+
SoftReference<EcodDatabase> ecodRef = versionedEcodDBs.get(version.toLowerCase());
74+
EcodDatabase ecod = null;
75+
if(ecodRef != null) {
76+
ecod = ecodRef.get();
77+
}
6678
if( ecod == null ) {
67-
logger.info("Creating new {}, version {}",EcodInstallation.class.getSimpleName(),version);
79+
logger.debug("Creating new {}, version {}",EcodInstallation.class.getSimpleName(),version);
6880
String cacheDir = new UserConfiguration().getCacheFilePath();
6981
ecod = new EcodInstallation(cacheDir, version);
70-
versionedEcodDBs.put(version.toLowerCase(), ecod);
82+
versionedEcodDBs.put(version.toLowerCase(), new SoftReference<EcodDatabase>(ecod));
7183

7284
// If the parsed version differed from that requested, add that too
7385
try {
7486
if( ! versionedEcodDBs.containsKey(ecod.getVersion().toLowerCase()) ) {
75-
versionedEcodDBs.put(ecod.getVersion().toLowerCase(),ecod);
87+
versionedEcodDBs.put(ecod.getVersion().toLowerCase(),new SoftReference<EcodDatabase>(ecod));
7688
}
7789
} catch (IOException e) {
7890
// For parsing errors, just use the requested version
7991
}
8092
}
93+
logger.trace("Releasing EcodFactory lock after getting version "+version);
8194

8295
return ecod;
8396
}
8497
}
98+
99+
private static void releaseReferences() {
100+
synchronized(versionedEcodDBs) {
101+
Iterator<Entry<String, SoftReference<EcodDatabase>>> it = versionedEcodDBs.entrySet().iterator();
102+
while(it.hasNext()) {
103+
Entry<String, SoftReference<EcodDatabase>> entry = it.next();
104+
SoftReference<EcodDatabase> ref = entry.getValue();
105+
if(ref.get() == null) {
106+
logger.debug("Removed version {} from EcodFactory to save memory.",entry.getKey());
107+
it.remove();
108+
}
109+
}
110+
}
111+
}
85112

86113
/**
87114
* Updates the default version

0 commit comments

Comments
 (0)