From f25436725e2dd2a0d3ddb672415b0991ce4f2ce0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 9 May 2008 14:44:28 +0000 Subject: [PATCH] 2008-05-09 Tatsuhiro Tsujikawa Added the ability to reuse connection in FTP and it is enabled by default. It can be disabled by --ftp-reuse-connection=false option. * src/Command.cc * src/DownloadCommand.cc * src/FtpDownloadCommand.cc * src/FtpDownloadCommand.h * src/FtpFinishDownloadCommand.cc * src/FtpFinishDownloadCommand.h * src/FtpInitiateConnectionCommand.cc * src/FtpNegotiationCommand.cc * src/FtpNegotiationCommand.h * src/HelpItemFactory.cc * src/Makefile.am * src/Makefile.in * src/OptionHandlerFactory.cc * src/option_processing.cc * src/prefs.h * src/usage_text.h --- ChangeLog | 21 ++++++ src/Command.cc | 2 +- src/DownloadCommand.cc | 18 +++-- src/FtpDownloadCommand.cc | 41 +++++++++-- src/FtpDownloadCommand.h | 7 +- src/FtpFinishDownloadCommand.cc | 102 ++++++++++++++++++++++++++++ src/FtpFinishDownloadCommand.h | 64 +++++++++++++++++ src/FtpInitiateConnectionCommand.cc | 15 ++-- src/FtpNegotiationCommand.cc | 19 ++++-- src/FtpNegotiationCommand.h | 8 ++- src/HelpItemFactory.cc | 5 ++ src/Makefile.am | 3 +- src/Makefile.in | 6 +- src/OptionHandlerFactory.cc | 1 + src/option_processing.cc | 5 ++ src/prefs.h | 2 + src/usage_text.h | 2 + 17 files changed, 290 insertions(+), 31 deletions(-) create mode 100644 src/FtpFinishDownloadCommand.cc create mode 100644 src/FtpFinishDownloadCommand.h diff --git a/ChangeLog b/ChangeLog index 237dd7bb..d837d215 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,24 @@ +2008-05-09 Tatsuhiro Tsujikawa + + Added the ability to reuse connection in FTP and it is enabled by + default. It can be disabled by --ftp-reuse-connection=false option. + * src/Command.cc + * src/DownloadCommand.cc + * src/FtpDownloadCommand.cc + * src/FtpDownloadCommand.h + * src/FtpFinishDownloadCommand.cc + * src/FtpFinishDownloadCommand.h + * src/FtpInitiateConnectionCommand.cc + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + * src/HelpItemFactory.cc + * src/Makefile.am + * src/Makefile.in + * src/OptionHandlerFactory.cc + * src/option_processing.cc + * src/prefs.h + * src/usage_text.h + 2008-05-09 Tatsuhiro Tsujikawa Call Command::transitStatus() before calling Command::execute(), diff --git a/src/Command.cc b/src/Command.cc index fa733e47..2ec3a401 100644 --- a/src/Command.cc +++ b/src/Command.cc @@ -52,7 +52,7 @@ void Command::transitStatus() break; default: status = STATUS_INACTIVE; - } + } } void Command::setStatus(STATUS status) diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index bd98519a..d2137dd6 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -178,13 +178,17 @@ bool DownloadCommand::prepareForNextSegment() { #endif // ENABLE_MESSAGE_DIGEST return true; } else { - SegmentHandle tempSegment = _segments.front(); - SegmentHandle nextSegment = - _requestGroup->getSegmentMan()->getSegment(cuid, - tempSegment->getIndex()+1); - if(!nextSegment.isNull() && nextSegment->getWrittenLength() == 0) { - e->commands.push_back(this); - return false; + if(_segments.size()) { + SegmentHandle tempSegment = _segments.front(); + SegmentHandle nextSegment = + _requestGroup->getSegmentMan()->getSegment(cuid, + tempSegment->getIndex()+1); + if(!nextSegment.isNull() && nextSegment->getWrittenLength() == 0) { + e->commands.push_back(this); + return false; + } else { + return prepareForRetry(0); + } } else { return prepareForRetry(0); } diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc index 87b78f02..c505e190 100644 --- a/src/FtpDownloadCommand.cc +++ b/src/FtpDownloadCommand.cc @@ -35,18 +35,47 @@ #include "FtpDownloadCommand.h" #include "Request.h" #include "Socket.h" +#include "Segment.h" +#include "DownloadEngine.h" +#include "RequestGroup.h" +#include "prefs.h" +#include "Option.h" +#include "FtpFinishDownloadCommand.h" +#include "FtpConnection.h" +#include "Logger.h" namespace aria2 { -FtpDownloadCommand::FtpDownloadCommand(int cuid, - const RequestHandle& req, - RequestGroup* requestGroup, - DownloadEngine* e, - const SocketHandle& dataSocket, - const SocketHandle& ctrlSocket) +FtpDownloadCommand::FtpDownloadCommand +(int cuid, + const RequestHandle& req, + RequestGroup* requestGroup, + const SharedHandle& ftpConnection, + DownloadEngine* e, + const SocketHandle& dataSocket, + const SocketHandle& ctrlSocket) :DownloadCommand(cuid, req, requestGroup, e, dataSocket), + _ftpConnection(ftpConnection), ctrlSocket(ctrlSocket) {} FtpDownloadCommand::~FtpDownloadCommand() {} + +bool FtpDownloadCommand::prepareForNextSegment() +{ + if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION) && + (uint64_t)_segments.front()->getPositionToWrite() == _requestGroup->getTotalLength()) { + Command* command = new FtpFinishDownloadCommand(cuid, req, _requestGroup, _ftpConnection, e, ctrlSocket); + e->commands.push_back(command); + + if(_requestGroup->downloadFinished()) { + // To run checksum checking, we had to call following function here. + DownloadCommand::prepareForNextSegment(); + } + return true; + } else { + return DownloadCommand::prepareForNextSegment(); + } +} + } // namespace aria2 diff --git a/src/FtpDownloadCommand.h b/src/FtpDownloadCommand.h index 3d257d9a..c4b2829e 100644 --- a/src/FtpDownloadCommand.h +++ b/src/FtpDownloadCommand.h @@ -39,15 +39,20 @@ namespace aria2 { -class SocketCore; +class FtpConnection; class FtpDownloadCommand : public DownloadCommand { private: + SharedHandle _ftpConnection; + SharedHandle ctrlSocket; +protected: + virtual bool prepareForNextSegment(); public: FtpDownloadCommand(int cuid, const SharedHandle& req, RequestGroup* requestGroup, + const SharedHandle& ftpConnection, DownloadEngine* e, const SharedHandle& dataSocket, const SharedHandle& ctrlSocket); diff --git a/src/FtpFinishDownloadCommand.cc b/src/FtpFinishDownloadCommand.cc new file mode 100644 index 00000000..d9a705e2 --- /dev/null +++ b/src/FtpFinishDownloadCommand.cc @@ -0,0 +1,102 @@ +/* */ +#include "FtpFinishDownloadCommand.h" +#include "Request.h" +#include "DownloadEngine.h" +#include "prefs.h" +#include "Option.h" +#include "FtpConnection.h" +#include "message.h" +#include "StringFormat.h" +#include "DlAbortEx.h" +#include "SocketCore.h" +#include "RequestGroup.h" +#include "Logger.h" + +namespace aria2 { + +FtpFinishDownloadCommand::FtpFinishDownloadCommand +(int cuid, + const RequestHandle& req, + RequestGroup* requestGroup, + const SharedHandle& ftpConnection, + DownloadEngine* e, + const SharedHandle& socket) + :AbstractCommand(cuid, req, requestGroup, e, socket), + _ftpConnection(ftpConnection) +{ + e->addSocketForReadCheck(socket, this); +} + +FtpFinishDownloadCommand::~FtpFinishDownloadCommand() +{ + e->deleteSocketForReadCheck(socket, this); +} + +// overrides AbstractCommand::execute(). +// AbstractCommand::_segments is empty. +bool FtpFinishDownloadCommand::execute() +{ + if(_requestGroup->isHaltRequested()) { + return true; + } + try { + unsigned int status = _ftpConnection->receiveResponse(); + if(status == 0) { + e->commands.push_back(this); + return false; + } + if(status != 226) { + throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); + } + if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { + std::pair peerInfo; + socket->getPeerInfo(peerInfo); + e->poolSocket(peerInfo.first, peerInfo.second, socket); + } + } catch(RecoverableException& e) { + logger->info(EX_EXCEPTION_CAUGHT, e); + } + if(_requestGroup->downloadFinished()) { + return true; + } else { + return prepareForRetry(0); + } +} + +// This function never be called. +bool FtpFinishDownloadCommand::executeInternal() { return true; } + +} // namespace aria2 diff --git a/src/FtpFinishDownloadCommand.h b/src/FtpFinishDownloadCommand.h new file mode 100644 index 00000000..2edc6d5b --- /dev/null +++ b/src/FtpFinishDownloadCommand.h @@ -0,0 +1,64 @@ +/* */ +#ifndef _D_FTP_FINISH_DOWNLOAD_COMMAND_H_ +#define _D_FTP_FINISH_DOWNLOAD_COMMAND_H_ + +#include "AbstractCommand.h" + +namespace aria2 { + +class FtpConnection; + +class FtpFinishDownloadCommand : public AbstractCommand { +private: + SharedHandle _ftpConnection; +protected: + virtual bool execute(); + + virtual bool executeInternal(); +public: + FtpFinishDownloadCommand(int cuid, + const SharedHandle& req, + RequestGroup* requestGroup, + const SharedHandle& ftpConnection, + DownloadEngine* e, + const SharedHandle& socket); + + virtual ~FtpFinishDownloadCommand(); +}; + +} // namespace aria2 + +#endif // _D_FTP_FINISH_DOWNLOAD_COMMAND_H_ diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index 1908483f..e5a94d56 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -80,10 +80,17 @@ Command* FtpInitiateConnectionCommand::createNextCommand throw DlAbortEx("ERROR"); } } else { - logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), - req->getPort()); - socket->establishConnection(resolvedAddresses.front(), req->getPort()); - command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); + SharedHandle pooledSocket = + e->popPooledSocket(resolvedAddresses, req->getPort()); + if(pooledSocket.isNull()) { + + logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), + req->getPort()); + socket->establishConnection(resolvedAddresses.front(), req->getPort()); + command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); + } else { + command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, FtpNegotiationCommand::SEQ_SEND_CWD); + } } return command; } diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 08b9a2d0..1d1870b8 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -64,17 +64,16 @@ FtpNegotiationCommand::FtpNegotiationCommand(int32_t cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e, - const SocketHandle& s): - AbstractCommand(cuid, req, requestGroup, e, s), sequence(SEQ_RECV_GREETING) + const SocketHandle& s, + Seq seq): + AbstractCommand(cuid, req, requestGroup, e, s), sequence(seq), + ftp(new FtpConnection(cuid, socket, req, e->option)) { - ftp = new FtpConnection(cuid, socket, req, e->option); disableReadCheckSocket(); setWriteCheckSocket(socket); } -FtpNegotiationCommand::~FtpNegotiationCommand() { - delete ftp; -} +FtpNegotiationCommand::~FtpNegotiationCommand() {} bool FtpNegotiationCommand::executeInternal() { while(processSequence(_segments.front())); @@ -82,7 +81,7 @@ bool FtpNegotiationCommand::executeInternal() { return prepareForRetry(0); } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { FtpDownloadCommand* command = - new FtpDownloadCommand(cuid, req, _requestGroup, e, dataSocket, socket); + new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); @@ -242,6 +241,11 @@ bool FtpNegotiationCommand::recvSize() { BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option)); if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) { sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; + // We can pool socket here + std::pair peerInfo; + socket->getPeerInfo(peerInfo); + e->poolSocket(peerInfo.first, peerInfo.second, socket); + return false; } _requestGroup->loadAndOpenFile(infoFile); @@ -250,6 +254,7 @@ bool FtpNegotiationCommand::recvSize() { sequence = SEQ_FILE_PREPARATION; disableReadCheckSocket(); + //setWriteCheckSocket(dataSocket); //e->noWait = true; diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index da6fe7d3..efefa836 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -43,7 +43,7 @@ class FtpConnection; class SocketCore; class FtpNegotiationCommand : public AbstractCommand { -private: +public: enum Seq { SEQ_RECV_GREETING, SEQ_SEND_USER, @@ -72,6 +72,7 @@ private: SEQ_DOWNLOAD_ALREADY_COMPLETED, SEQ_FILE_PREPARATION }; +private: bool recvGreeting(); bool sendUser(); bool recvUser(); @@ -100,7 +101,7 @@ private: SharedHandle dataSocket; SharedHandle serverSocket; Seq sequence; - FtpConnection* ftp; + SharedHandle ftp; protected: virtual bool executeInternal(); public: @@ -108,7 +109,8 @@ public: const SharedHandle& req, RequestGroup* requestGroup, DownloadEngine* e, - const SharedHandle& s); + const SharedHandle& s, + Seq seq = SEQ_RECV_GREETING); virtual ~FtpNegotiationCommand(); }; diff --git a/src/HelpItemFactory.cc b/src/HelpItemFactory.cc index 5f42888e..7fb2b33c 100644 --- a/src/HelpItemFactory.cc +++ b/src/HelpItemFactory.cc @@ -465,6 +465,11 @@ TagContainerHandle HelpItemFactory::createHelpItems(const Option* op) tc->addItem(item); } #endif // ENABLE_ASYNC_DNS + { + HelpItemHandle item(new HelpItem(PREF_FTP_REUSE_CONNECTION, TEXT_FTP_REUSE_CONNECTION, op->get(PREF_FTP_REUSE_CONNECTION))); + item->addTag(TAG_FTP); + tc->addItem(item); + } { HelpItemHandle item(new HelpItem("help", TEXT_HELP, TAG_BASIC)); item->setAvailableValues diff --git a/src/Makefile.am b/src/Makefile.am index a5e107e1..8d48390d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -185,7 +185,8 @@ SRCS = Socket.h\ NullStatCalc.h\ StringFormat.cc StringFormat.h\ HttpNullDownloadCommand.cc HttpNullDownloadCommand.h\ - InitiateConnectionCommand.cc InitiateConnectionCommand.h + InitiateConnectionCommand.cc InitiateConnectionCommand.h\ + FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h if ENABLE_ASYNC_DNS SRCS += AsyncNameResolver.cc AsyncNameResolver.h diff --git a/src/Makefile.in b/src/Makefile.in index 384f906f..b65a14d4 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -408,6 +408,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ NullStatCalc.h StringFormat.cc StringFormat.h \ HttpNullDownloadCommand.cc HttpNullDownloadCommand.h \ InitiateConnectionCommand.cc InitiateConnectionCommand.h \ + FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ AsyncNameResolver.cc AsyncNameResolver.h \ IteratableChunkChecksumValidator.cc \ IteratableChunkChecksumValidator.h \ @@ -787,7 +788,8 @@ am__objects_15 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ HelpItemFactory.$(OBJEXT) SingleFileDownloadContext.$(OBJEXT) \ TimedHaltCommand.$(OBJEXT) ProtocolDetector.$(OBJEXT) \ StringFormat.$(OBJEXT) HttpNullDownloadCommand.$(OBJEXT) \ - InitiateConnectionCommand.$(OBJEXT) $(am__objects_1) \ + InitiateConnectionCommand.$(OBJEXT) \ + FtpFinishDownloadCommand.$(OBJEXT) $(am__objects_1) \ $(am__objects_2) $(am__objects_3) $(am__objects_4) \ $(am__objects_5) $(am__objects_6) $(am__objects_7) \ $(am__objects_8) $(am__objects_9) $(am__objects_10) \ @@ -1130,6 +1132,7 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ NullStatCalc.h StringFormat.cc StringFormat.h \ HttpNullDownloadCommand.cc HttpNullDownloadCommand.h \ InitiateConnectionCommand.cc InitiateConnectionCommand.h \ + FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ $(am__append_1) $(am__append_2) $(am__append_3) \ $(am__append_4) $(am__append_5) $(am__append_6) \ $(am__append_7) $(am__append_8) $(am__append_9) \ @@ -1374,6 +1377,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FinMetalinkParserState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnection.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpDownloadCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpFinishDownloadCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpInitiateConnectionCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpNegotiationCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 9baa9bea..16de9a2d 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -131,6 +131,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() #ifdef ENABLE_ASYNC_DNS handlers.push_back(SH(new BooleanOptionHandler(PREF_ASYNC_DNS))); #endif // ENABLE_ASYNC_DNS + handlers.push_back(SH(new BooleanOptionHandler(PREF_FTP_REUSE_CONNECTION))); return handlers; } diff --git a/src/option_processing.cc b/src/option_processing.cc index 234f9581..6240b0d2 100644 --- a/src/option_processing.cc +++ b/src/option_processing.cc @@ -149,6 +149,7 @@ Option* createDefaultOption() #else op->put(PREF_ASYNC_DNS, V_FALSE); #endif // ENABLE_ASYNC_DNS + op->put(PREF_FTP_REUSE_CONNECTION, V_TRUE); return op; } @@ -223,6 +224,7 @@ Option* option_processing(int argc, char* const argv[]) #ifdef ENABLE_ASYNC_DNS { PREF_ASYNC_DNS, optional_argument, &lopt, 216 }, #endif // ENABLE_ASYNC_DNS + { PREF_FTP_REUSE_CONNECTION, optional_argument, &lopt, 217 }, #if defined ENABLE_BITTORRENT || ENABLE_METALINK { PREF_SHOW_FILES, no_argument, NULL, 'S' }, { PREF_SELECT_FILE, required_argument, &lopt, 21 }, @@ -429,6 +431,9 @@ Option* option_processing(int argc, char* const argv[]) cmdstream << PREF_ASYNC_DNS << "=" << toBoolArg(optarg) << "\n"; break; #endif // ENABLE_ASYNC_DNS + case 217: + cmdstream << PREF_FTP_REUSE_CONNECTION << "=" << toBoolArg(optarg) << "\n"; + break; } break; } diff --git a/src/prefs.h b/src/prefs.h index 7515b18b..f243107d 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -146,6 +146,8 @@ # define V_TUNNEL "tunnel" // values: true | false #define PREF_FTP_PASV "ftp-pasv" +// values: true | false +#define PREF_FTP_REUSE_CONNECTION "ftp-reuse-connection" /** * HTTP related preferences diff --git a/src/usage_text.h b/src/usage_text.h index 128bb06e..8397745a 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -338,3 +338,5 @@ _(" --header=HEADER Append HEADER to HTTP request header. You can u _(" -q, --quiet[=true|false] Make aria2 quite (no console output).") #define TEXT_ASYNC_DNS \ _(" --async-dns[=true|false] Enable asynchronous DNS.") +#define TEXT_FTP_REUSE_CONNECTION \ +_(" --ftp-reuse-connection[=true|false] Reuse connection in FTP.")