Magic-Mirror/aria2
1
0
Fork 0
mirror of https://github.com/aria2/aria2.git synced 2024-12-14 01:18:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

2008-09-10 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Browse Source 
Added --max-file-not-found=NUM option. If aria2 receives `file 
not
	found' error from HTTP/FTP server NUM times without getting 
single
	bytes of data, then force the download to fail.
	Basically aria2 tris all URIs available. So if long list of URIs 
are
	specified in Metalink, then aria2 will take a long time to fail
	eventually if all URIs are outdated. This option eases this 
situation
	to limit the number of servers to try.
	* src/FtpNegotiationCommand.cc
	* src/HelpItemFactory.cc
	* src/HttpResponseCommand.cc
	* src/OptionHandlerFactory.cc
	* src/PeerStat.h
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/SegmentMan.cc
	* src/SegmentMan.h
	* src/option_processing.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
This commit is contained in:
Tatsuhiro Tsujikawa 2008-09-10 14:56:44 +00:00
parent 1b2a76706f
commit 3599067ec1
14 changed files with 112 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
ChangeLog
View File

@ -1,3 +1,26 @@
2008-09-10 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Added --max-file-not-found=NUM option. If aria2 receives `file not
found' error from HTTP/FTP server NUM times without getting single
bytes of data, then force the download to fail.
Basically aria2 tris all URIs available. So if long list of URIs are
specified in Metalink, then aria2 will take a long time to fail
eventually if all URIs are outdated. This option eases this situation
to limit the number of servers to try.
* src/FtpNegotiationCommand.cc
* src/HelpItemFactory.cc
* src/HttpResponseCommand.cc
* src/OptionHandlerFactory.cc
* src/PeerStat.h
* src/RequestGroup.cc
* src/RequestGroup.h
* src/SegmentMan.cc
* src/SegmentMan.h
* src/option_processing.cc
* src/prefs.cc
* src/prefs.h
* src/usage_text.h
2008-09-10 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Fixed the bug that bulkReceiveResponse() reads all received data as a

2
src/FtpNegotiationCommand.cc
View File

@ -204,6 +204,7 @@ bool FtpNegotiationCommand::recvCwd() {
}
if(status != 250) {
poolConnection();
_requestGroup->increaseAndValidateFileNotFoundCount();
throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
}
if(e->option->getAsBool(PREF_REMOTE_TIME)) {
@ -444,6 +445,7 @@ bool FtpNegotiationCommand::recvRetr() {
return false;
}
if(status != 150 && status != 125) {
_requestGroup->increaseAndValidateFileNotFoundCount();
throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
}
if(e->option->getAsBool(PREF_FTP_PASV)) {

8
src/HelpItemFactory.cc
View File

@ -526,6 +526,14 @@ TagContainerHandle HelpItemFactory::createHelpItems(const Option* op)
item->addTag(TAG_ADVANCED);
tc->addItem(item);
}
{
HelpItemHandle item(new HelpItem(PREF_MAX_FILE_NOT_FOUND,
TEXT_MAX_FILE_NOT_FOUND,
op->get(PREF_MAX_FILE_NOT_FOUND)));
item->addTag(TAG_HTTP);
item->addTag(TAG_FTP);
tc->addItem(item);
}
{
HelpItemHandle item(new HelpItem("help", TEXT_HELP, TAG_BASIC));
item->setAvailableValues

4
src/HttpResponseCommand.cc
View File

@ -91,9 +91,11 @@ bool HttpResponseCommand::executeInternal()
httpResponse->retrieveCookie();
if(httpResponse->getResponseStatus() >= HttpHeader::S300) {
if(httpResponse->getResponseStatus() == HttpHeader::S404) {
_requestGroup->increaseAndValidateFileNotFoundCount();
}
return skipResponseBody(httpResponse);
}
if(!_requestGroup->isSingleHostMultiConnectionEnabled()) {
// Query by hostname. Searching by CUID may returns NULL.
// In case when resuming download, ServerHost is registered with CUID A.

1
src/OptionHandlerFactory.cc
View File

@ -156,6 +156,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
handlers.push_back(SH(new DefaultOptionHandler(PREF_SERVER_STAT_OF)));
handlers.push_back(SH(new BooleanOptionHandler(PREF_REMOTE_TIME)));
handlers.push_back(SH(new NumberOptionHandler(PREF_CONNECT_TIMEOUT, 1, 600)));
handlers.push_back(SH(new NumberOptionHandler(PREF_MAX_FILE_NOT_FOUND, 0)));
return handlers;
}

22
src/PeerStat.h
View File

@ -59,6 +59,8 @@ private:
PeerStat::STATUS status;
unsigned int _avgDownloadSpeed;
unsigned int _avgUploadSpeed;
uint64_t _sessionDownloadLength;
uint64_t _sessionUploadLength;
public:
PeerStat(int32_t cuid, const std::string& hostname,
@ -68,11 +70,15 @@ public:
_protocol(protocol),
status(PeerStat::IDLE),
_avgDownloadSpeed(0),
_avgUploadSpeed(0) {}
_avgUploadSpeed(0),
_sessionDownloadLength(0),
_sessionUploadLength(0) {}
PeerStat(int32_t cuid = 0):cuid(cuid), status(PeerStat::IDLE),
_avgDownloadSpeed(0),
_avgUploadSpeed(0) {}
_avgUploadSpeed(0),
_sessionDownloadLength(0),
_sessionUploadLength(0) {}
~PeerStat() {}
@ -107,10 +113,12 @@ public:
void updateDownloadLength(size_t bytes) {
downloadSpeed.update(bytes);
_sessionDownloadLength += bytes;
}
void updateUploadLength(size_t bytes) {
uploadSpeed.update(bytes);
_sessionUploadLength += bytes;
}
unsigned int getMaxDownloadSpeed() const {
@ -172,6 +180,16 @@ public:
{
return _protocol;
}
uint64_t getSessionDownloadLength() const
{
return _sessionDownloadLength;
}
uint64_t getSessionUploadLength() const
{
return _sessionUploadLength;
}
};
typedef SharedHandle<PeerStat> PeerStatHandle;

13
src/RequestGroup.cc
View File

@ -124,6 +124,7 @@ RequestGroup::RequestGroup(const Option* option,
_singleHostMultiConnectionEnabled(true),
_uriSelector(new InOrderURISelector()),
_lastModifiedTime(-1),
_fileNotFoundCount(0),
_option(option),
_logger(LogFactory::getInstance())
{
@ -1037,6 +1038,7 @@ void RequestGroup::applyLastModifiedTimeToLocalFiles()
_logger->info("Last-Modified attrs of %zu files were updated.", n);
}
}
void RequestGroup::updateLastModifiedTime(const Time& time)
{
if(time.good() && _lastModifiedTime < time) {
@ -1044,4 +1046,15 @@ void RequestGroup::updateLastModifiedTime(const Time& time)
}
}
void RequestGroup::increaseAndValidateFileNotFoundCount()
{
++_fileNotFoundCount;
const unsigned int maxCount = _option->getAsInt(PREF_MAX_FILE_NOT_FOUND);
if(maxCount > 0 && _fileNotFoundCount >= maxCount &&
_segmentMan->calculateSessionDownloadLength() == 0) {
throw DownloadFailureException
(StringFormat("Reached max-file-not-found count=%u", maxCount).str());
}
}
} // namespace aria2

4
src/RequestGroup.h
View File

@ -118,6 +118,8 @@ private:
Time _lastModifiedTime;
unsigned int _fileNotFoundCount;
const Option* _option;
Logger* _logger;
@ -368,6 +370,8 @@ public:
void applyLastModifiedTimeToLocalFiles();
void updateLastModifiedTime(const Time& time);
void increaseAndValidateFileNotFoundCount();
};
typedef SharedHandle<RequestGroup> RequestGroupHandle;

16
src/SegmentMan.cc
View File

@ -46,6 +46,7 @@
#include "DownloadContext.h"
#include "Piece.h"
#include <algorithm>
#include <numeric>
#include <cassert>
namespace aria2 {
@ -306,6 +307,21 @@ unsigned int SegmentMan::calculateDownloadSpeed() const {
return speed;
}
class PeerStatDownloadLengthOperator {
public:
uint64_t operator()(uint64_t total, const SharedHandle<PeerStat>& ps)
{
return ps->getSessionDownloadLength()+total;
}
};
uint64_t SegmentMan::calculateSessionDownloadLength() const
{
return std::accumulate(peerStats.begin(), peerStats.end(), 0LL,
PeerStatDownloadLengthOperator());
}
size_t SegmentMan::countFreePieceFrom(size_t index) const
{
size_t numPieces = _downloadContext->getNumPieces();

5
src/SegmentMan.h
View File

@ -179,6 +179,11 @@ public:
*/
unsigned int calculateDownloadSpeed() const;
/**
* Returns the downloaded bytes in this session.
*/
uint64_t calculateSessionDownloadLength() const;
size_t countFreePieceFrom(size_t index) const;
};

5
src/option_processing.cc
View File

@ -160,6 +160,7 @@ Option* createDefaultOption()
op->put(PREF_URI_SELECTOR, V_INORDER);
op->put(PREF_SERVER_STAT_TIMEOUT, "86400");// 1day
op->put(PREF_REMOTE_TIME, V_FALSE);
op->put(PREF_MAX_FILE_NOT_FOUND, "0");
return op;
}
@ -243,6 +244,7 @@ Option* option_processing(int argc, char* const argv[])
{ PREF_SERVER_STAT_TIMEOUT.c_str(), required_argument, &lopt, 223 },
{ PREF_REMOTE_TIME.c_str(), optional_argument, 0, 'R' },
{ PREF_CONNECT_TIMEOUT.c_str(), required_argument, &lopt, 224 },
{ PREF_MAX_FILE_NOT_FOUND.c_str(), required_argument, &lopt, 225 },
#if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
{ PREF_SHOW_FILES.c_str(), no_argument, NULL, 'S' },
{ PREF_SELECT_FILE.c_str(), required_argument, &lopt, 21 },
@ -488,6 +490,9 @@ Option* option_processing(int argc, char* const argv[])
case 224:
cmdstream << PREF_CONNECT_TIMEOUT << "=" << optarg << "\n";
break;
case 225:
cmdstream << PREF_MAX_FILE_NOT_FOUND << "=" << optarg << "\n";
break;
}
break;
}

2
src/prefs.cc
View File

@ -149,6 +149,8 @@ const std::string PREF_SERVER_STAT_IF("server-stat-if");
const std::string PREF_SERVER_STAT_OF("server-stat-of");
// value: true | false
const std::string PREF_REMOTE_TIME("remote-time");
// value: 1*digit
const std::string PREF_MAX_FILE_NOT_FOUND("max-file-not-found");
/**
* FTP related preferences

2
src/prefs.h
View File

@ -153,6 +153,8 @@ extern const std::string PREF_SERVER_STAT_IF;
extern const std::string PREF_SERVER_STAT_OF;
// value: true | false
extern const std::string PREF_REMOTE_TIME;
// value: 1*digit
extern const std::string PREF_MAX_FILE_NOT_FOUND;
/**
* FTP related preferences

8
src/usage_text.h
View File

@ -365,3 +365,11 @@ _(" --connect-timeout=SEC Set the connect timeout in seconds to establish
" connection to HTTP/FTP/proxy server. After the\n"\
" connection is established, this option makes no\n"\
" effect and --timeout option is used instead.")
#define TEXT_MAX_FILE_NOT_FOUND \
_(" --max-file-not-found=NUM If aria2 recieves `file not found' status from the\n"\
" remote HTTP/FTP servers NUM times without getting\n"\
" a single byte, then force the download to fail.\n"\
" Specify 0 to disable this option.\n"\
" This options is only effective only when using\n"\
" HTTP/FTP servers.")