much cleaner replica set management and fixed some bugs in failover

erh · erh · commit 06fc09c29984 · 2010-10-02T08:24:11.000-04:00
diff --git a/src/main/com/mongodb/DBTCPConnector.java b/src/main/com/mongodb/DBTCPConnector.java
@@ -40,7 +40,7 @@ public DBTCPConnector( Mongo m , ServerAddress addr )
 
         if ( addr.isPaired() ){
             _allHosts = new ArrayList<ServerAddress>( addr.explode() );
-            _createLogger.info( "switching to replica set mode : " + _allHosts + " -> " + _curAddress  );
+            _createLogger.info( "switching to replica set mode : " + _allHosts + " -> " + _curMaster  );
         }
         else {
             _set( addr );
@@ -62,7 +62,7 @@ public DBTCPConnector( Mongo m , List<ServerAddress> all )
 
         _allHosts = new ArrayList<ServerAddress>( all ); // make a copy so it can't be modified
 
-        _createLogger.info( all  + " -> " + _curAddress );
+        _createLogger.info( all  + " -> " + _curMaster );
     }
 
     private static ServerAddress _checkAddress( ServerAddress addr ){
@@ -204,7 +204,7 @@ public Response call( DB db , DBCollection coll , OutMessage m , int retries )
         ServerError err = res.getError();
         
         if ( err != null && err.isNotMasterError() ){
-            _pickCurrent();
+            _findMaster( true );
             if ( retries <= 0 ){
                 throw new MongoException( "not talking to master and retries used up" );
             }
@@ -216,33 +216,41 @@ public Response call( DB db , DBCollection coll , OutMessage m , int retries )
     }
 
     public ServerAddress getAddress(){
-        return _curAddress;
+        return _curMaster;
     }
 
     public List<ServerAddress> getAllAddress() {
         return _allHosts;
     }
 
     public String getConnectPoint(){
-        return _curAddress.toString();
+        return _curMaster.toString();
     }
 
     boolean _error( Throwable t )
         throws MongoException {
         if ( _allHosts != null ){
             _logger.log( Level.WARNING , "replica set mode, switching master" , t );
-            _pickCurrent();
+            _findMaster( true );
         }
         return true;
     }
 
     class MyPort {
 
         DBPort get( boolean keep ){
-            if ( _port != null )
-                return _port;
+            _findMaster( false );
+
+            if ( _port != null ){
+                if ( _pool == _curPortPool )
+                    return _port;
+                _pool.done( _port );
+                _port = null;
+                _pool = null;
+            }
             
-            DBPort p = _curPortPool.get();
+            _pool = _curPortPool;
+            DBPort p = _pool.get();
             if ( keep && _inRequest )
                 _port = p;
 
@@ -251,11 +259,11 @@ DBPort get( boolean keep ){
         
         void done( DBPort p ){
             if ( p != _port )
-                _curPortPool.done( p );
+                _pool.done( p );
         }
 
         void error( DBPort p , Exception e ){
-            _curPortPool.done( p );
+            _pool.done( p );
             p.close();
 
             _port = null;
@@ -270,7 +278,7 @@ void requestEnsureConnection(){
             if ( _port != null )
                 return;
 
-            _port = _curPortPool.get();
+            _port = _pool.get();
         }
 
         void requestStart(){
@@ -279,24 +287,28 @@ void requestStart(){
 
         void requestDone(){
             if ( _port != null )
-                _curPortPool.done( _port );
+                _pool.done( _port );
             _port = null;
             _inRequest = false;
         }
 
         DBPort _port;
+        DBPortPool _pool;
         boolean _inRequest;
     }
     
     /**
      * @return next to try
      */
-    ServerAddress _addAllFromSet( DBObject o ){
-        Object foo = o.get( "hosts" );
+    ServerAddress _addAllFromSet( CommandResult res ){
+        if ( res == null )
+            return null;
+        
+        Object foo = res.get( "hosts" );
         if ( ! ( foo instanceof List ) )
             return null;
 
-        String primary = (String)o.get("primary");
+        String primary = (String)res.get("primary");
         
         ServerAddress primaryAddress = null;
 
@@ -322,89 +334,86 @@ ServerAddress _addAllFromSet( DBObject o ){
 
         return primaryAddress;
     }
-    
-    void _pickInitial()
-        throws MongoException {
-        if ( _curAddress != null )
-            return;
 
-        // we need to just get a server to query for ismaster
-        _pickCurrent();
 
+    /**
+     * this will update set meta data
+     * @return if addr should be the new master
+     */
+    boolean _findMasterLoop( ServerAddress addr , ServerAddress[] outPrimary ){
         try {
-            _logger.info( "current address beginning of _pickInitial: " + _curAddress );
-            
-            DBObject im = isMasterCmd();
+            CommandResult res = _isMasterCmd( addr );
+            if ( res == null )
+                return false;
             
-            ServerAddress other = _addAllFromSet( im );
-            
-            if ( _isMaster( im ) ) 
-                return;
+            ServerAddress prim = _addAllFromSet( res );
+            if ( outPrimary != null )
+                outPrimary[0] = prim;
 
-            if ( other != null ){
-                _set( other );
-                im = isMasterCmd();
-                _addAllFromSet( im );                
-                if ( _isMaster( im ) )
+            return _isMaster( res );
+        }
+        catch ( Exception e ){
+            _logger.warning( "can't do _findMasterLoop " + addr + "\t" + e );
+            return false;
+        }
+            
+    }
+    
+    void _findMaster( boolean reset )
+        throws MongoException {
+        
+        if ( _allHosts == null )
+            return;
+        
+        if ( ! reset && _curMaster != null )
+            return;
+        
+        synchronized ( _allHosts ){
+            if ( _curMaster != null ){
+                if ( _findMasterLoop( _curMaster , null  ) )
                     return;
-
-                _logger.severe( "primary given was wrong: " + other + " going to scan" );
+                
+                _curMaster = null;
+                _curPortPool = null;
             }
             
-            synchronized ( _allHosts ){
+            for ( int loopNumber=0; loopNumber<3; loopNumber++ ){
+                
                 Collections.shuffle( _allHosts );
-                for ( ServerAddress a : _allHosts ){
-                    if ( _curAddress == a )
-                        continue;
 
-                    _logger.info( "remote [" + _curAddress + "] -> [" + a + "]" );
-                    _set( a );
+                ServerAddress newPrimary[] = new ServerAddress[1];
+
+                for ( int i=0; i<_allHosts.size(); i++ ){
+                    ServerAddress addr = _allHosts.get(i);
+                    _logger.config( "_findMaster looking at : " + addr );
+                    newPrimary[0] = null;
                     
-                    im = isMasterCmd();
-                    _addAllFromSet( im );
-                    if ( _isMaster( im ) )
+                    if ( _findMasterLoop( addr , newPrimary ) ){
+                        _set( addr );
                         return;
+                    }
                     
-                    if ( _allHosts.size() == 2 )
-                        _logger.severe( "switched to: " + a + " but isn't master" );
-                }
-                
-                throw new MongoException( "can't find master" );
-            }
-        }
-        catch ( Exception e ){
-            _logger.log( Level.SEVERE , "can't pick initial master, using random one" , e );
-        }
-    }
-
-    private void _pickCurrent()
-        throws MongoException {
-        if ( _allHosts == null )
-            throw new MongoException( "got master/slave issue but not in master/slave mode on the client side" );
-
-        synchronized ( _allHosts ){
-            Collections.shuffle( _allHosts );
-            for ( int i=0; i<_allHosts.size(); i++ ){
-                ServerAddress a = _allHosts.get( i );
-                if ( a == _curAddress )
-                    continue;
-
-                if ( _curAddress != null ){
-                    _logger.info( "switching from [" + _curAddress + "] to [" + a + "]" );
+                    if ( newPrimary[0] != null ){
+                        addr = newPrimary[0];
+                        if ( _findMasterLoop( addr , null ) ){
+                            _set( addr );
+                            return;
+                        }
+                    }
                 }
 
-                _set( a );
-                return;
             }
+            // sleep - waiting for system to recover
+            try { Thread.sleep( 1000 ); } catch ( Exception e ){}
         }
-
-        throw new MongoException( "couldn't find a new host to swtich too" );
+                
+        throw new MongoException( "can't find master" );
     }
 
     private boolean _set( ServerAddress addr ){
-        if ( _curAddress == addr )
+        if ( _curMaster == addr )
             return false;
-        _curAddress = addr;
+        _curMaster = addr;
         _curPortPool = _portHolder.get( addr );
         return true;
     }
@@ -414,26 +423,34 @@ public String debugString(){
         if ( _allHosts != null )
             buf.append( "replica set : " ).append( _allHosts );
         else
-            buf.append( _curAddress ).append( " " ).append( _curAddress._addr );
+            buf.append( _curMaster ).append( " " ).append( _curMaster._addr );
 
         return buf.toString();
     }
 
-    DBObject isMasterCmd(){
-        DBCollection collection = _mongo.getDB( "admin" ).getCollection( "$cmd" );
-        
-        Iterator<DBObject> i = collection.__find( _isMaster , null , 0 , 1 , 0 );
-        if ( i == null || ! i.hasNext() )
-            throw new MongoException( "no result for ismaster query?" );
-        
-        DBObject res = i.next();
-        if ( i.hasNext() )
-            throw new MongoException( "what's going on" );
-        
-        return res;
+    CommandResult _isMasterCmd( ServerAddress addr ){
+        DBPortPool pool = _portHolder.get( addr );
+        DBPort p = null;
+        try {
+            p = pool.get();
+            CommandResult res = p.runCommand( _mongo.getDB( "admin" ) , _isMaster );
+            return res;
+        }
+        catch ( Exception e ){
+            _logger.log( Level.INFO , "can't run ismaster on : " + addr + "\t" + e );
+            return null;
+        }
+        finally {
+            if ( p != null ){
+                pool.done( p );
+            }
+        }
     }
 
-    boolean _isMaster( DBObject res ){
+    boolean _isMaster( CommandResult res ){
+        if ( res == null )
+            return false;
+
         Object x = res.get( "ismaster" );
         if ( x == null )
             throw new IllegalStateException( "ismaster shouldn't be null: " + res );
@@ -452,7 +469,7 @@ public void close(){
     }
 
     final Mongo _mongo;
-    private ServerAddress _curAddress;
+    private ServerAddress _curMaster;
     private DBPortPool _curPortPool;
     private DBPortPool.Holder _portHolder;
     private final List<ServerAddress> _allHosts;
diff --git a/src/main/com/mongodb/Mongo.java b/src/main/com/mongodb/Mongo.java
@@ -163,7 +163,7 @@ public Mongo( ServerAddress addr , MongoOptions options )
         _addrs = null;
         _options = options;
         _connector = new DBTCPConnector( this , _addr );
-        _connector._pickInitial();
+        _connector._findMaster( false );
     }
 
     /**
@@ -187,7 +187,7 @@ public Mongo( ServerAddress left , ServerAddress right , MongoOptions options )
         _addrs = Arrays.asList( left , right );
         _options = options;
         _connector = new DBTCPConnector( this , _addrs );
-        _connector._pickInitial();
+        _connector._findMaster( false );
     }
 
     /**
@@ -212,8 +212,7 @@ public Mongo( List<ServerAddress> replicaSetSeeds , MongoOptions options )
         _addrs = replicaSetSeeds;
         _options = options;
         _connector = new DBTCPConnector( this , _addrs );
-        _connector._pickInitial();
-
+        _connector._findMaster( false );
     }
 
     public DB getDB( String dbname ){
diff --git a/src/test/com/mongodb/ReplSetTest.java b/src/test/com/mongodb/ReplSetTest.java
@@ -22,6 +22,9 @@ public void run(){
                     Thread.sleep( 500 );
                     _coll.findOne();
                 }
+                catch ( NullPointerException n ){
+                    n.printStackTrace();
+                }
                 catch ( Exception e ){
                     System.out.println( _a + "\t" + e );
                 }

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,9 @@ public void run(){`
`22`	`22`	`Thread.sleep( 500 );`
`23`	`23`	`_coll.findOne();`
`24`	`24`	`}`
	`25`	`+ catch ( NullPointerException n ){`
	`26`	`+ n.printStackTrace();`
	`27`	`+ }`
`25`	`28`	`catch ( Exception e ){`
`26`	`29`	`System.out.println( _a + "\t" + e );`
`27`	`30`	`}`