2008-05-09 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Added the ability to reuse connection in FTP and it is enabled 
by
	default. It can be disabled by --ftp-reuse-connection=false 
option.
	* src/Command.cc
	* src/DownloadCommand.cc
	* src/FtpDownloadCommand.cc
	* src/FtpDownloadCommand.h
	* src/FtpFinishDownloadCommand.cc
	* src/FtpFinishDownloadCommand.h
	* src/FtpInitiateConnectionCommand.cc
	* src/FtpNegotiationCommand.cc
	* src/FtpNegotiationCommand.h
	* src/HelpItemFactory.cc
	* src/Makefile.am
	* src/Makefile.in
	* src/OptionHandlerFactory.cc
	* src/option_processing.cc
	* src/prefs.h
	* src/usage_text.h

ChangeLog

@@ -1,3 +1,24 @@
+2008-05-09  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Added the ability to reuse connection in FTP and it is enabled by
+	default. It can be disabled by --ftp-reuse-connection=false option.
+	* src/Command.cc
+	* src/DownloadCommand.cc
+	* src/FtpDownloadCommand.cc
+	* src/FtpDownloadCommand.h
+	* src/FtpFinishDownloadCommand.cc
+	* src/FtpFinishDownloadCommand.h
+	* src/FtpInitiateConnectionCommand.cc
+	* src/FtpNegotiationCommand.cc
+	* src/FtpNegotiationCommand.h
+	* src/HelpItemFactory.cc
+	* src/Makefile.am
+	* src/Makefile.in
+	* src/OptionHandlerFactory.cc
+	* src/option_processing.cc
+	* src/prefs.h
+	* src/usage_text.h
+
 2008-05-09  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Call Command::transitStatus() before calling Command::execute(),

src/Command.cc

@@ -52,7 +52,7 @@ void Command::transitStatus()
     break;
   default:
     status = STATUS_INACTIVE;
-    }
+  }
 }
 
 void Command::setStatus(STATUS status)

src/DownloadCommand.cc

@@ -178,13 +178,17 @@ bool DownloadCommand::prepareForNextSegment() {
 #endif // ENABLE_MESSAGE_DIGEST
     return true;
   } else {
-    SegmentHandle tempSegment = _segments.front();
-    SegmentHandle nextSegment =
-      _requestGroup->getSegmentMan()->getSegment(cuid,
-						 tempSegment->getIndex()+1);
-    if(!nextSegment.isNull() && nextSegment->getWrittenLength() == 0) {
-      e->commands.push_back(this);
-      return false;
+    if(_segments.size()) {
+      SegmentHandle tempSegment = _segments.front();
+      SegmentHandle nextSegment =
+	_requestGroup->getSegmentMan()->getSegment(cuid,
+						   tempSegment->getIndex()+1);
+      if(!nextSegment.isNull() && nextSegment->getWrittenLength() == 0) {
+	e->commands.push_back(this);
+	return false;
+      } else {
+	return prepareForRetry(0);
+      }
     } else {
       return prepareForRetry(0);
     }

src/FtpDownloadCommand.cc

@@ -35,18 +35,47 @@
 #include "FtpDownloadCommand.h"
 #include "Request.h"
 #include "Socket.h"
+#include "Segment.h"
+#include "DownloadEngine.h"
+#include "RequestGroup.h"
+#include "prefs.h"
+#include "Option.h"
+#include "FtpFinishDownloadCommand.h"
+#include "FtpConnection.h"
+#include "Logger.h"
 
 namespace aria2 {
 
-FtpDownloadCommand::FtpDownloadCommand(int cuid,
-				       const RequestHandle& req,
-				       RequestGroup* requestGroup,
-				       DownloadEngine* e,
-				       const SocketHandle& dataSocket,
-				       const SocketHandle& ctrlSocket)
+FtpDownloadCommand::FtpDownloadCommand
+(int cuid,
+ const RequestHandle& req,
+ RequestGroup* requestGroup,
+ const SharedHandle<FtpConnection>& ftpConnection,
+ DownloadEngine* e,
+ const SocketHandle& dataSocket,
+ const SocketHandle& ctrlSocket)
   :DownloadCommand(cuid, req, requestGroup, e, dataSocket),
+   _ftpConnection(ftpConnection),
    ctrlSocket(ctrlSocket) {}
 
 FtpDownloadCommand::~FtpDownloadCommand() {}
 
+
+bool FtpDownloadCommand::prepareForNextSegment()
+{
+  if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION) &&
+     (uint64_t)_segments.front()->getPositionToWrite() == _requestGroup->getTotalLength()) {
+    Command* command = new FtpFinishDownloadCommand(cuid, req, _requestGroup, _ftpConnection, e, ctrlSocket);
+    e->commands.push_back(command);
+
+    if(_requestGroup->downloadFinished()) {
+      // To run checksum checking, we had to call following function here.
+      DownloadCommand::prepareForNextSegment();
+    }
+    return true;
+  } else {
+    return DownloadCommand::prepareForNextSegment();
+  }
+}
+
 } // namespace aria2

src/FtpDownloadCommand.h

@@ -39,15 +39,20 @@
 
 namespace aria2 {
 
-class SocketCore;
+class FtpConnection;
 
 class FtpDownloadCommand : public DownloadCommand {
 private:
+  SharedHandle<FtpConnection> _ftpConnection;
+
   SharedHandle<SocketCore> ctrlSocket;
+protected:
+  virtual bool prepareForNextSegment();
 public:
   FtpDownloadCommand(int cuid,
 		     const SharedHandle<Request>& req,
 		     RequestGroup* requestGroup,
+		     const SharedHandle<FtpConnection>& ftpConnection,
 		     DownloadEngine* e,
 		     const SharedHandle<SocketCore>& dataSocket,
 		     const SharedHandle<SocketCore>& ctrlSocket);

src/FtpFinishDownloadCommand.cc

@@ -0,0 +1,102 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "FtpFinishDownloadCommand.h"
+#include "Request.h"
+#include "DownloadEngine.h"
+#include "prefs.h"
+#include "Option.h"
+#include "FtpConnection.h"
+#include "message.h"
+#include "StringFormat.h"
+#include "DlAbortEx.h"
+#include "SocketCore.h"
+#include "RequestGroup.h"
+#include "Logger.h"
+
+namespace aria2 {
+
+FtpFinishDownloadCommand::FtpFinishDownloadCommand
+(int cuid,
+ const RequestHandle& req,
+ RequestGroup* requestGroup,
+ const SharedHandle<FtpConnection>& ftpConnection,
+ DownloadEngine* e,
+ const SharedHandle<SocketCore>& socket)
+  :AbstractCommand(cuid, req, requestGroup, e, socket),
+   _ftpConnection(ftpConnection)
+{
+  e->addSocketForReadCheck(socket, this);
+}
+
+FtpFinishDownloadCommand::~FtpFinishDownloadCommand()
+{
+  e->deleteSocketForReadCheck(socket, this);
+}
+
+// overrides AbstractCommand::execute().
+// AbstractCommand::_segments is empty.
+bool FtpFinishDownloadCommand::execute()
+{
+  if(_requestGroup->isHaltRequested()) {
+    return true;
+  }
+  try {
+    unsigned int status = _ftpConnection->receiveResponse();
+    if(status == 0) {
+      e->commands.push_back(this);
+      return false;
+    }
+    if(status != 226) {
+      throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
+    }
+    if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) {
+      std::pair<std::string, uint16_t> peerInfo;
+      socket->getPeerInfo(peerInfo);
+      e->poolSocket(peerInfo.first, peerInfo.second, socket);
+    }
+  } catch(RecoverableException& e) {
+    logger->info(EX_EXCEPTION_CAUGHT, e);
+  }
+  if(_requestGroup->downloadFinished()) {
+    return true;
+  } else {
+    return prepareForRetry(0);
+  }
+}
+
+// This function never be called.
+bool FtpFinishDownloadCommand::executeInternal() { return true; }
+
+} // namespace aria2

src/FtpFinishDownloadCommand.h

@@ -0,0 +1,64 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_FTP_FINISH_DOWNLOAD_COMMAND_H_
+#define _D_FTP_FINISH_DOWNLOAD_COMMAND_H_
+
+#include "AbstractCommand.h"
+
+namespace aria2 {
+
+class FtpConnection;
+
+class FtpFinishDownloadCommand : public AbstractCommand {
+private:
+  SharedHandle<FtpConnection> _ftpConnection;
+protected:
+  virtual bool execute();
+
+  virtual bool executeInternal();
+public:
+  FtpFinishDownloadCommand(int cuid,
+			   const SharedHandle<Request>& req,
+			   RequestGroup* requestGroup,
+			   const SharedHandle<FtpConnection>& ftpConnection,
+			   DownloadEngine* e,
+			   const SharedHandle<SocketCore>& socket);
+
+  virtual ~FtpFinishDownloadCommand();
+};
+
+} // namespace aria2
+
+#endif // _D_FTP_FINISH_DOWNLOAD_COMMAND_H_

src/FtpInitiateConnectionCommand.cc

@@ -80,10 +80,17 @@ Command* FtpInitiateConnectionCommand::createNextCommand
       throw DlAbortEx("ERROR");
     }
   } else {
-    logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(),
-		 req->getPort());
-    socket->establishConnection(resolvedAddresses.front(), req->getPort());
-    command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket);
+    SharedHandle<SocketCore> pooledSocket =
+      e->popPooledSocket(resolvedAddresses, req->getPort());
+    if(pooledSocket.isNull()) {
+
+      logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(),
+		   req->getPort());
+      socket->establishConnection(resolvedAddresses.front(), req->getPort());
+      command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket);
+    } else {
+      command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, FtpNegotiationCommand::SEQ_SEND_CWD);
+    }
   }
   return command;
 }

src/FtpNegotiationCommand.cc

@@ -64,17 +64,16 @@ FtpNegotiationCommand::FtpNegotiationCommand(int32_t cuid,
 					     const RequestHandle& req,
 					     RequestGroup* requestGroup,
 					     DownloadEngine* e,
-					     const SocketHandle& s):
-  AbstractCommand(cuid, req, requestGroup, e, s), sequence(SEQ_RECV_GREETING)
+					     const SocketHandle& s,
+					     Seq seq):
+  AbstractCommand(cuid, req, requestGroup, e, s), sequence(seq),
+  ftp(new FtpConnection(cuid, socket, req, e->option))
 {
-  ftp = new FtpConnection(cuid, socket, req, e->option);
   disableReadCheckSocket();
   setWriteCheckSocket(socket);
 }
 
-FtpNegotiationCommand::~FtpNegotiationCommand() {
-  delete ftp;
-}
+FtpNegotiationCommand::~FtpNegotiationCommand() {}
 
 bool FtpNegotiationCommand::executeInternal() {
   while(processSequence(_segments.front()));
@@ -82,7 +81,7 @@ bool FtpNegotiationCommand::executeInternal() {
     return prepareForRetry(0);
   } else if(sequence == SEQ_NEGOTIATION_COMPLETED) {
     FtpDownloadCommand* command =
-      new FtpDownloadCommand(cuid, req, _requestGroup, e, dataSocket, socket);
+      new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket);
     command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT));
     command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME));
     command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT));
@@ -242,6 +241,11 @@ bool FtpNegotiationCommand::recvSize() {
     BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option));
     if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) {
       sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED;
+      // We can pool socket here
+      std::pair<std::string, uint16_t> peerInfo;
+      socket->getPeerInfo(peerInfo);
+      e->poolSocket(peerInfo.first, peerInfo.second, socket);
+
       return false;
     }
     _requestGroup->loadAndOpenFile(infoFile);
@@ -250,6 +254,7 @@ bool FtpNegotiationCommand::recvSize() {
     
     sequence = SEQ_FILE_PREPARATION;
     disableReadCheckSocket();
+
     //setWriteCheckSocket(dataSocket);
 
     //e->noWait = true;

src/FtpNegotiationCommand.h

@@ -43,7 +43,7 @@ class FtpConnection;
 class SocketCore;
 
 class FtpNegotiationCommand : public AbstractCommand {
-private:
+public:
   enum Seq {
     SEQ_RECV_GREETING,
     SEQ_SEND_USER,
@@ -72,6 +72,7 @@ private:
     SEQ_DOWNLOAD_ALREADY_COMPLETED,
     SEQ_FILE_PREPARATION
   };
+private:
   bool recvGreeting();
   bool sendUser();
   bool recvUser();
@@ -100,7 +101,7 @@ private:
   SharedHandle<SocketCore> dataSocket;
   SharedHandle<SocketCore> serverSocket;
   Seq sequence;
-  FtpConnection* ftp;
+  SharedHandle<FtpConnection> ftp;
 protected:
   virtual bool executeInternal();
 public:
@@ -108,7 +109,8 @@ public:
 			const SharedHandle<Request>& req,
 			RequestGroup* requestGroup,
 			DownloadEngine* e,
-			const SharedHandle<SocketCore>& s);
+			const SharedHandle<SocketCore>& s,
+			Seq seq = SEQ_RECV_GREETING);
   virtual ~FtpNegotiationCommand();
 };
 

src/HelpItemFactory.cc

@@ -465,6 +465,11 @@ TagContainerHandle HelpItemFactory::createHelpItems(const Option* op)
     tc->addItem(item);
   }
 #endif // ENABLE_ASYNC_DNS
+  {
+    HelpItemHandle item(new HelpItem(PREF_FTP_REUSE_CONNECTION, TEXT_FTP_REUSE_CONNECTION, op->get(PREF_FTP_REUSE_CONNECTION)));
+    item->addTag(TAG_FTP);
+    tc->addItem(item);
+  }
   {
     HelpItemHandle item(new HelpItem("help", TEXT_HELP, TAG_BASIC));
     item->setAvailableValues

src/Makefile.am

@@ -185,7 +185,8 @@ SRCS =  Socket.h\
 	NullStatCalc.h\
 	StringFormat.cc StringFormat.h\
 	HttpNullDownloadCommand.cc HttpNullDownloadCommand.h\
-	InitiateConnectionCommand.cc InitiateConnectionCommand.h
+	InitiateConnectionCommand.cc InitiateConnectionCommand.h\
+	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h
 
 if ENABLE_ASYNC_DNS
 SRCS += AsyncNameResolver.cc AsyncNameResolver.h

src/Makefile.in

@@ -408,6 +408,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	NullStatCalc.h StringFormat.cc StringFormat.h \
 	HttpNullDownloadCommand.cc HttpNullDownloadCommand.h \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
+	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
 	AsyncNameResolver.cc AsyncNameResolver.h \
 	IteratableChunkChecksumValidator.cc \
 	IteratableChunkChecksumValidator.h \
@@ -787,7 +788,8 @@ am__objects_15 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	HelpItemFactory.$(OBJEXT) SingleFileDownloadContext.$(OBJEXT) \
 	TimedHaltCommand.$(OBJEXT) ProtocolDetector.$(OBJEXT) \
 	StringFormat.$(OBJEXT) HttpNullDownloadCommand.$(OBJEXT) \
-	InitiateConnectionCommand.$(OBJEXT) $(am__objects_1) \
+	InitiateConnectionCommand.$(OBJEXT) \
+	FtpFinishDownloadCommand.$(OBJEXT) $(am__objects_1) \
 	$(am__objects_2) $(am__objects_3) $(am__objects_4) \
 	$(am__objects_5) $(am__objects_6) $(am__objects_7) \
 	$(am__objects_8) $(am__objects_9) $(am__objects_10) \
@@ -1130,6 +1132,7 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	NullStatCalc.h StringFormat.cc StringFormat.h \
 	HttpNullDownloadCommand.cc HttpNullDownloadCommand.h \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
+	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
 	$(am__append_1) $(am__append_2) $(am__append_3) \
 	$(am__append_4) $(am__append_5) $(am__append_6) \
 	$(am__append_7) $(am__append_8) $(am__append_9) \
@@ -1374,6 +1377,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FinMetalinkParserState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnection.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpDownloadCommand.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpFinishDownloadCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpInitiateConnectionCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpNegotiationCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@

src/OptionHandlerFactory.cc

@@ -131,6 +131,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
 #ifdef ENABLE_ASYNC_DNS
   handlers.push_back(SH(new BooleanOptionHandler(PREF_ASYNC_DNS)));
 #endif // ENABLE_ASYNC_DNS
+  handlers.push_back(SH(new BooleanOptionHandler(PREF_FTP_REUSE_CONNECTION)));
   return handlers;
 }
 

src/option_processing.cc

@@ -149,6 +149,7 @@ Option* createDefaultOption()
 #else
   op->put(PREF_ASYNC_DNS, V_FALSE);
 #endif // ENABLE_ASYNC_DNS
+  op->put(PREF_FTP_REUSE_CONNECTION, V_TRUE);
   return op;
 }
 
@@ -223,6 +224,7 @@ Option* option_processing(int argc, char* const argv[])
 #ifdef ENABLE_ASYNC_DNS
       { PREF_ASYNC_DNS, optional_argument, &lopt, 216 },
 #endif // ENABLE_ASYNC_DNS
+      { PREF_FTP_REUSE_CONNECTION, optional_argument, &lopt, 217 },
 #if defined ENABLE_BITTORRENT || ENABLE_METALINK
       { PREF_SHOW_FILES, no_argument, NULL, 'S' },
       { PREF_SELECT_FILE, required_argument, &lopt, 21 },
@@ -429,6 +431,9 @@ Option* option_processing(int argc, char* const argv[])
 	cmdstream << PREF_ASYNC_DNS << "=" << toBoolArg(optarg) << "\n";
 	break;
 #endif // ENABLE_ASYNC_DNS
+      case 217:
+	cmdstream << PREF_FTP_REUSE_CONNECTION << "=" << toBoolArg(optarg) << "\n";
+	break;
       }
       break;
     }

src/prefs.h

@@ -146,6 +146,8 @@
 #  define V_TUNNEL "tunnel"
 // values: true | false
 #define PREF_FTP_PASV "ftp-pasv"
+// values: true | false
+#define PREF_FTP_REUSE_CONNECTION "ftp-reuse-connection"
 
 /**
  * HTTP related preferences

src/usage_text.h

@@ -338,3 +338,5 @@ _(" --header=HEADER              Append HEADER to HTTP request header. You can u
 _(" -q, --quiet[=true|false]     Make aria2 quite (no console output).")
 #define TEXT_ASYNC_DNS \
 _(" --async-dns[=true|false]     Enable asynchronous DNS.")
+#define TEXT_FTP_REUSE_CONNECTION \
+_(" --ftp-reuse-connection[=true|false] Reuse connection in FTP.")