biojava · sbliven · Jun 18, 2018 · Jun 4, 2018 · Jun 4, 2018 · Jun 8, 2018
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/StructureIO.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/StructureIO.java
@@ -119,6 +119,11 @@ private static void checkInitAtomCache() {
 	public static void setAtomCache(AtomCache c){
 		cache = c;
 	}
+
+	public static AtomCache getAtomCache() {
+		checkInitAtomCache();
+		return cache;
+	}
 
 	/**
 	 * Returns the first biologicalAssembly that is available for a protein structure. For more documentation on quaternary structures see:

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/cath/CathInstallation.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/cath/CathInstallation.java
@@ -636,6 +636,7 @@ private void parseCathDomall(BufferedReader bufferedReader) throws IOException{
 
 	protected void downloadFileFromRemote(URL remoteURL, File localFile) throws IOException{
 //        System.out.println("downloading " + remoteURL + " to: " + localFile);
+		LOGGER.info("Downloading file {} to local file {}", remoteURL, localFile);
 
 		long timeS = System.currentTimeMillis();
 		File tempFile  = File.createTempFile(FileDownloadUtils.getFilePrefix(localFile), "."+ FileDownloadUtils.getFileExtension(localFile));
@@ -665,7 +666,7 @@ protected void downloadFileFromRemote(URL remoteURL, File localFile) throws IOEx
 			disp = disp / 1024.0;
 		}
 		long timeE = System.currentTimeMillis();
-		LOGGER.info("Downloaded file {} ({}) to local file {} in {} sec.", remoteURL, String.format("%.1f",disp) + unit, localFile, (timeE - timeS)/1000);
+		LOGGER.info("Downloaded {} in {} sec. to {}", String.format("%.1f",disp) + unit, (timeE - timeS)/1000, localFile);
 	}
 
 	private boolean domainDescriptionFileAvailable(){

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/LocalPDBDirectory.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/LocalPDBDirectory.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.nio.file.Files;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -127,6 +128,9 @@ public static enum FetchBehavior {
 
 	protected static final String lineSplit = System.getProperty("file.separator");
 
+	/** Minimum size for a valid structure file (CIF or PDB), in bytes */
+	public static final long MIN_PDB_FILE_SIZE = 40;  // Empty gzip files are 20bytes. Add a few more for buffer.
+
 	private File path;
 	private List<String> extensions;
 
@@ -402,8 +406,9 @@ public void prefetchStructure(String pdbId) throws IOException {
 	 * Attempts to delete all versions of a structure from the local directory.
 	 * @param pdbId
 	 * @return True if one or more files were deleted
+	 * @throws IOException if the file cannot be deleted
 	 */
-	public boolean deleteStructure(String pdbId){
+	public boolean deleteStructure(String pdbId) throws IOException{
 		boolean deleted = false;
 		// Force getLocalFile to check in obsolete locations
 		ObsoleteBehavior obsolete = getObsoleteBehavior();
@@ -421,7 +426,7 @@ public boolean deleteStructure(String pdbId){
 				// delete file
 				boolean success = existing.delete();
 				if(success) {
-					logger.info("Deleting "+existing.getAbsolutePath());
+					logger.debug("Deleting "+existing.getAbsolutePath());
 				}
 				deleted = deleted || success;
 
@@ -430,7 +435,7 @@ public boolean deleteStructure(String pdbId){
 				if(parent != null) {
 					success = parent.delete();
 					if(success) {
-						logger.info("Deleting "+parent.getAbsolutePath());
+						logger.debug("Deleting "+parent.getAbsolutePath());
 					}
 				}
 
@@ -660,8 +665,9 @@ protected File getDir(String pdbId, boolean obsolete) {
 	 * Searches for previously downloaded files
 	 * @param pdbId
 	 * @return A file pointing to the existing file, or null if not found
+	 * @throws IOException If the file exists but is empty and can't be deleted
 	 */
-	public File getLocalFile(String pdbId) {
+	public File getLocalFile(String pdbId) throws IOException {
 
 		// Search for existing files
 
@@ -687,6 +693,11 @@ public File getLocalFile(String pdbId) {
 				for(String ex : getExtensions() ){
 					File f = new File(searchdir,prefix + pdbId.toLowerCase() + ex) ;
 					if ( f.exists()) {
+						// delete files that are too short to have contents
+						if( f.length() < MIN_PDB_FILE_SIZE ) {
+							Files.delete(f.toPath());
+							return null;
+						}
 						return f;
 					}
 				}
@@ -697,9 +708,11 @@ public File getLocalFile(String pdbId) {
 	}
 
 	protected boolean checkFileExists(String pdbId){
-		File path =  getLocalFile(pdbId);
-		if ( path != null)
-			return true;
+		try {
+			File path =  getLocalFile(pdbId);
+			if ( path != null)
+				return true;
+		} catch(IOException e) {}
 		return false;
 	}
 

diff --git a/...ava-structure/src/main/java/org/biojava/nbio/structure/io/mmcif/ChemCompGroupFactory.java b/...ava-structure/src/main/java/org/biojava/nbio/structure/io/mmcif/ChemCompGroupFactory.java
@@ -68,9 +68,8 @@ public static ChemComp getChemComp(String recordName){
 	 * again. Note that this change can have unexpected behavior of
 	 * code executed afterwards.
 	 * <p>
-	 * Changing the provider does not reset the cache, so Chemical
-	 * Component definitions already downloaded from previous providers
-	 * will be used. To reset the cache see {@link #getCache()).
+	 * Changing the provider also resets the cache, so any groups
+	 * previously accessed will be reread or re-downloaded.
 	 *
 	 * @param provider
 	 */
@@ -84,6 +83,15 @@ public static void setChemCompProvider(ChemCompProvider provider) {
 	public static ChemCompProvider getChemCompProvider(){
 		return chemCompProvider;
 	}
+
+	/**
+	 * Force the in-memory cache to be reset.
+	 * 
+	 * Note that the ChemCompProvider may have additional memory or disk caches that need to be cleared too.
+	 */
+	public static void clearCache() {
+		cache.clear();
+	}
 
 	public static Group getGroupFromChemCompDictionary(String recordName) {
 

diff --git a/...structure/src/main/java/org/biojava/nbio/structure/io/mmcif/DownloadChemCompProvider.java b/...structure/src/main/java/org/biojava/nbio/structure/io/mmcif/DownloadChemCompProvider.java
@@ -42,6 +42,7 @@
 import org.biojava.nbio.core.util.InputStreamProvider;
 import org.biojava.nbio.structure.align.util.HTTPConnectionTools;
 import org.biojava.nbio.structure.align.util.UserConfiguration;
+import org.biojava.nbio.structure.io.LocalPDBDirectory;
 import org.biojava.nbio.structure.io.mmcif.model.ChemComp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -85,32 +86,37 @@ public class DownloadChemCompProvider implements ChemCompProvider {
 		protectedIDs.add("AUX");
 		protectedIDs.add("NUL");
 	}
+
+	private static ChemCompProvider fallback = null; // Fallback provider if the download fails
 
 	/** by default we will download only some of the files. User has to request that all files should be downloaded...
 	 *
 	 */
 	boolean downloadAll = false;
 
 	public DownloadChemCompProvider(){
-		logger.debug("Initialising DownloadChemCompProvider");
-
-		// note that path is static, so this is just to make sure that all non-static methods will have path initialised
-		initPath();
+		this(null);
 	}
 
 	public DownloadChemCompProvider(String cacheFilePath){
 		logger.debug("Initialising DownloadChemCompProvider");
 
 		// note that path is static, so this is just to make sure that all non-static methods will have path initialised
-		path = new File(cacheFilePath);
+		if(cacheFilePath != null) {
+			path = new File(cacheFilePath);
+		}
 	}
 
-	private static void initPath(){
-
+	/**
+	 * Get this provider's cache path
+	 * @return
+	 */
+	public static File getPath(){
 		if (path==null) {
 			UserConfiguration config = new UserConfiguration();
 			path = new File(config.getCacheFilePath());
 		}
+		return path;
 	}
 
 	/**
@@ -127,7 +133,7 @@ public void checkDoFirstInstall(){
 
 		// this makes sure there is a file separator between every component,
 		// if path has a trailing file separator or not, it will work for both cases
-		File dir = new File(path, CHEM_COMP_CACHE_DIRECTORY);
+		File dir = new File(getPath(), CHEM_COMP_CACHE_DIRECTORY);
 		File f = new File(dir, "components.cif.gz");
 
 		if ( ! f.exists()) {
@@ -161,7 +167,7 @@ private void split() throws IOException {
 
 		logger.info("Installing individual chem comp files ...");
 
-		File dir = new File(path, CHEM_COMP_CACHE_DIRECTORY);
+		File dir = new File(getPath(), CHEM_COMP_CACHE_DIRECTORY);
 		File f = new File(dir, "components.cif.gz");
 
 
@@ -212,7 +218,7 @@ private void split() throws IOException {
 	 */
 	private void writeID(String contents, String currentID) throws IOException{
 
-		String localName = DownloadChemCompProvider.getLocalFileName(currentID);
+		String localName = getLocalFileName(currentID);
 
 		try ( PrintWriter pw = new PrintWriter(new GZIPOutputStream(new FileOutputStream(localName))) ) {
 
@@ -272,7 +278,10 @@ public  ChemComp getChemComp(String recordName) {
 
 				ChemComp chemComp = dict.getChemComp(recordName);
 
-				return chemComp;
+				// May be null if the file was corrupt. Fall back on ReducedChemCompProvider in that case
+				if(chemComp != null) {
+					return chemComp;
+				}
 
 			} catch (IOException e) {
 
@@ -296,9 +305,12 @@ public  ChemComp getChemComp(String recordName) {
 
 		// see https://github.com/biojava/biojava/issues/315
 		// probably a network error happened. Try to use the ReducedChemCOmpProvider
-		ReducedChemCompProvider reduced = new ReducedChemCompProvider();
+		if( fallback == null) {
+			fallback = new ReducedChemCompProvider();
+		}
 
-		return reduced.getChemComp(recordName);
+		logger.warn("Falling back to ReducedChemCompProvider for {}. This could indicate a network error.", recordName);
+		return fallback.getChemComp(recordName);
 
 	}
 
@@ -313,16 +325,15 @@ public static String getLocalFileName(String recordName){
 			recordName = "_" + recordName;
 		}
 
-		initPath();
-
-		File f = new File(path, CHEM_COMP_CACHE_DIRECTORY);
+		File f = new File(getPath(), CHEM_COMP_CACHE_DIRECTORY);
 		if (! f.exists()){
 			logger.info("Creating directory " + f);
 
 			boolean success = f.mkdir();
 			// we've checked in initPath that path is writable, so there's no need to check if it succeeds
 			// in the unlikely case that in the meantime it isn't writable at least we log an error
-			if (!success) logger.error("Directory {} could not be created",f);
+			if (!success)
+				logger.error("Directory {} could not be created",f);
 
 		}
 
@@ -337,6 +348,14 @@ private static  boolean fileExists(String recordName){
 
 		File f = new File(fileName);
 
+		// delete files that are too short to have contents
+		if( f.length() < LocalPDBDirectory.MIN_PDB_FILE_SIZE ) {
+			// Delete defensively.
+			// Note that if delete is unsuccessful, we re-download the file anyways
+			f.delete();
+			return false;
+		}
+
 		return f.exists();
 
 	}

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/mmcif/SimpleMMcifParser.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/mmcif/SimpleMMcifParser.java
@@ -212,6 +212,9 @@ public void parse(BufferedReader buf)
 
 		// the first line is a data_PDBCODE line, test if this looks like a mmcif file
 		line = buf.readLine();
+		while( line != null && (line.isEmpty() || line.startsWith(COMMENT_CHAR))) {
+			line = buf.readLine();
+		}
 		if (line == null || !line.startsWith(MMCIF_TOP_HEADER)){
 			logger.error("This does not look like a valid mmCIF file! The first line should start with 'data_', but is: '" + line+"'");
 			triggerDocumentEnd();

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/util/FileDownloadUtils.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/util/FileDownloadUtils.java
@@ -21,9 +21,6 @@
  */
 package org.biojava.nbio.structure.io.util;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -36,6 +33,15 @@
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.channels.ReadableByteChannel;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class FileDownloadUtils {
 
@@ -240,6 +246,41 @@ public static URLConnection prepareURLConnection(String url, int timeout) throws
 		connection.setConnectTimeout(timeout);
 		return connection;
 	}
+
+	/**
+	 * Recursively delete a folder & contents
+	 *
+	 * @param dir directory to delete
+	 */
+	public static void deleteDirectory(Path dir) throws IOException {
+		if(dir == null || !Files.exists(dir))
+			return;
+		Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
+	        @Override
+	        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+	            Files.delete(file);
+	            return FileVisitResult.CONTINUE;
+	        }
+
+	        @Override
+	        public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
+	            if (e != null) {
+	                throw e;
+	            }
+	            Files.delete(dir);
+	            return FileVisitResult.CONTINUE;
+	        }
+	    });
+	}
+	/**
+	 * Recursively delete a folder & contents
+	 *
+	 * @param dir directory to delete
+	 */
+	public static void deleteDirectory(String dir) throws IOException {
+		deleteDirectory(Paths.get(dir));
+	}
+
 
 	public static void main(String[] args) {
 		String url;

diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/TestAtomCache.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/TestAtomCache.java
@@ -46,7 +46,7 @@ public class TestAtomCache {
 	private AtomCache cache;
 
 	@Before
-	public void setUp() {
+	public void setUp() throws IOException {
 		cache = new AtomCache();
 
 		// Delete files which were cached in previous tests

diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/TestExperimentalTechniques.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/TestExperimentalTechniques.java
@@ -31,7 +31,7 @@
 public class TestExperimentalTechniques {
 
 	@Test
-	public void test4LNC() throws IOException, StructureException {
+	public void test6F2Q() throws IOException, StructureException {
 
 		// a multiple experimental techniques PDB entry (X-RAY + NEUTRON DIFFRACTION)
 
@@ -40,9 +40,9 @@ public void test4LNC() throws IOException, StructureException {
 		StructureIO.setAtomCache(cache);
 
 		cache.setUseMmCif(false);
-		Structure sPdb = StructureIO.getStructure("4LNC");
+		Structure sPdb = StructureIO.getStructure("6F2Q");
 		cache.setUseMmCif(true);
-		Structure sCif = StructureIO.getStructure("4LNC");
+		Structure sCif = StructureIO.getStructure("6F2Q");
 
 		comparePdbToCif(sPdb, sCif);