diff --git a/ChangeLog b/ChangeLog index 04756041..c7cbe795 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,8 @@ * main.cc: The value of --http-auth-scheme option is chagned from 'BASIC' to 'basic' * main.cc: Added --timeout command-line option. + * main.cc: Added --min-segment-size command-line option. + * main.cc: Added --max-retries command-line option. * prefs.h: option string constants are now defined in prefs.h 2006-02-19 Tatsuhiro Tsujikawa diff --git a/TODO b/TODO index 90d5a5f9..2435e4b0 100644 --- a/TODO +++ b/TODO @@ -4,3 +4,4 @@ * Add SSL client cert support * Better HTTP status handling * Download files listed in a specifed file. +* check MD5 checksum \ No newline at end of file diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index a16bf6a6..6fe1d2ae 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -102,7 +102,8 @@ bool AbstractCommand::execute() { delete(err); //req->resetUrl(); req->addTryCount(); - if(req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRY)) { + if(e->option->getAsInt(PREF_MAX_TRIES) != 0 && + req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRIES)) { e->logger->error(MSG_MAX_TRY, cuid, req->getTryCount()); return true; } else { diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index f8eaa97d..b3d2a70b 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -47,8 +47,8 @@ public: queue commands; SegmentMan* segmentMan; DiskWriter* diskWriter; - Logger* logger; - Option* option; + const Logger* logger; + const Option* option; DownloadEngine(); ~DownloadEngine(); diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 1d478f1b..54844e74 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -110,8 +110,12 @@ void FtpConnection::sendRetr() const { int FtpConnection::getStatus(string response) const { int status; - // TODO we must handle when the response is not like "%d %*s" - // In this case, we return 0 + // When the response is not like "%d %*s", + // we return 0. + if(response.find_first_not_of("0123456789") != 3 + || response.find(" ") != 3) { + return 0; + } if(sscanf(response.c_str(), "%d %*s", &status) == 1) { return status; } else { diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index f5788be7..4f808a8b 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -65,10 +65,13 @@ bool HttpResponseCommand::executeInternal(Segment seg) { return handleDefaultEncoding(headers); } } else { - // TODO we must check headers["size"] == e->segmentMan->totalSize here if(req->getFile() != e->segmentMan->filename) { throw new DlAbortEx(EX_FILENAME_MISMATCH, req->getFile().c_str(), e->segmentMan->filename.c_str()); } + long long int size = headers.getFirstAsLLInt("Content-Length"); + if(e->segmentMan->totalSize != size) { + throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); + } createHttpDownloadCommand(); return true; } @@ -99,9 +102,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { bool segFileExists = e->segmentMan->segmentFileExists(); e->segmentMan->downloadStarted = true; if(segFileExists) { - e->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - // we must check headers["size"] == e->segmentMan->totalSize here if(e->segmentMan->totalSize != size) { return new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); } diff --git a/src/Option.cc b/src/Option.cc index 96576b83..35e4a35c 100644 --- a/src/Option.cc +++ b/src/Option.cc @@ -50,3 +50,12 @@ int Option::getAsInt(const string& name) const { return (int)strtol((*itr).second.c_str(), NULL, 10); } } + +long long int Option::getAsLLInt(const string& name) const { + map::const_iterator itr = table.find(name); + if(itr == table.end()) { + return 0; + } else { + return (int)strtoll((*itr).second.c_str(), NULL, 10); + } +} diff --git a/src/Option.h b/src/Option.h index fa1e0c67..7cacef50 100644 --- a/src/Option.h +++ b/src/Option.h @@ -38,6 +38,7 @@ public: bool defined(const string& name) const; string get(const string& name) const; int getAsInt(const string& name) const; + long long int getAsLLInt(const string& name) const; }; #endif // _D_OPTION_H_ diff --git a/src/Request.cc b/src/Request.cc index 6eb4aa1e..9d183cd4 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -79,14 +79,12 @@ bool Request::parseUrl(string url) { Util::split(hostAndPort, url.substr(hp, hep-hp), ':'); host = hostAndPort.first; if(hostAndPort.second != "") { - // TODO rewrite this using strtoul function. If strtoul fails, - // return false. port = (int)strtol(hostAndPort.second.c_str(), NULL, 10); if(!(0 < port && port <= 65535)) { return false; } } else { - // If port is not specified, then we leave it 0. + // If port is not specified, then we set it to default port of its protocol.. port = defPort; } string::size_type direp = url.find_last_of("/"); diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 579c2bfe..0089575a 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -28,6 +28,7 @@ #include "Util.h" #include "File.h" #include "message.h" +#include "prefs.h" SegmentMan::SegmentMan():totalSize(0),isSplittable(true),downloadStarted(false),dir(".") {} @@ -85,7 +86,7 @@ bool SegmentMan::getSegment(Segment& seg, int cuid) { if(s.finish) { continue; } - if(s.ep-(s.sp+s.ds) > 524288) { + if(s.ep-(s.sp+s.ds) > option->getAsLLInt(PREF_MIN_SEGMENT_SIZE)) { long long int nep = (s.ep-(s.sp+s.ds))/2+(s.sp+s.ds); //nseg = { cuid, nep+1, s.ep, 0, false }; seg.cuid = cuid; diff --git a/src/SegmentMan.h b/src/SegmentMan.h index e971a176..075453ff 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -26,6 +26,7 @@ #include "common.h" #include "Logger.h" #include "Segment.h" +#include "Option.h" using namespace std; @@ -81,6 +82,7 @@ public: string ufilename; const Logger* logger; + const Option* option; SegmentMan(); ~SegmentMan(); diff --git a/src/main.cc b/src/main.cc index 63c46a95..a352a674 100644 --- a/src/main.cc +++ b/src/main.cc @@ -101,40 +101,45 @@ void showUsage() { cout << "Usage: " << PACKAGE_NAME << " [options] URL ..." << endl; cout << endl; cout << "Options:" << endl; - cout << " -d, --dir=DIR The directory to store downloaded file." << endl; - cout << " -o, --out=FILE The file name for downloaded file." << endl; - cout << " -l, --log=LOG The file path to store log. If '-' is specified," << endl; - cout << " log is written to stdout." << endl; - cout << " -D, --daemon Run as daemon." << endl; - cout << " -s, --split=N Download a file using s connections. s must be" << endl; - cout << " between 1 and 5. If this option is specified the" << endl; - cout << " first URL is used, and the other URLs are ignored." << endl; - cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; - cout << " for errors. Specify a value between 0 and 60." << endl; - cout << " -t, --timeout=SEC Set timeout in second." << endl; - cout << " --http-proxy=HOST:PORT Use HTTP proxy server. This affects to all" << endl; - cout << " URLs." << endl; - cout << " --http-user=USER Set HTTP user. This affects to all URLs." << endl; - cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; - cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; - cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; - cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; - cout << " is the only supported scheme. You MUST specify" << endl; - cout << " this option in order to use HTTP authentication" << endl; - cout << " as well as --http-proxy option." << endl; - cout << " --referer=REFERER Set Referer. This affects to all URLs." << endl; - cout << " --ftp-user=USER Set FTP user. This affects to all URLs." << endl; - cout << " Default: anonymous" << endl; - cout << " --ftp-passwd=PASSWD Set FTP password. This affects to all URLs." << endl; - cout << " Default: ARIA2USER@" << endl; - cout << " --ftp-type=TYPE Set FTP transfer type. TYPE is either 'binary'" << endl; - cout << " or 'ascii'." << endl; - cout << " Default: binary" << endl; - cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; - cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; - cout << " 'tunnel'." << endl; - cout << " -v, --version Print the version number and exit." << endl; - cout << " -h, --help Print this message and exit." << endl; + cout << " -d, --dir=DIR The directory to store downloaded file." << endl; + cout << " -o, --out=FILE The file name for downloaded file." << endl; + cout << " -l, --log=LOG The file path to store log. If '-' is specified," << endl; + cout << " log is written to stdout." << endl; + cout << " -D, --daemon Run as daemon." << endl; + cout << " -s, --split=N Download a file using s connections. s must be" << endl; + cout << " between 1 and 5. If this option is specified the" << endl; + cout << " first URL is used, and the other URLs are ignored." << endl; + cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; + cout << " for errors. Specify a value between 0 and 60." << endl; + cout << " Default: 5" << endl; + cout << " -t, --timeout=SEC Set timeout in second. Default: 60" << endl; + cout << " -m, --max-tries=N Set number of tries. 0 means unlimited." << endl; + cout << " Default: 5" << endl; + cout << " --min-segment-size=SIZE[K|M] Set minimum segment size. You can append" << endl; + cout << " K or M(1K = 1024, 1M = 1024K)." << endl; + cout << " --http-proxy=HOST:PORT Use HTTP proxy server. This affects to all" << endl; + cout << " URLs." << endl; + cout << " --http-user=USER Set HTTP user. This affects to all URLs." << endl; + cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; + cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; + cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; + cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; + cout << " is the only supported scheme. You MUST specify" << endl; + cout << " this option in order to use HTTP authentication" << endl; + cout << " as well as --http-proxy option." << endl; + cout << " --referer=REFERER Set Referer. This affects to all URLs." << endl; + cout << " --ftp-user=USER Set FTP user. This affects to all URLs." << endl; + cout << " Default: anonymous" << endl; + cout << " --ftp-passwd=PASSWD Set FTP password. This affects to all URLs." << endl; + cout << " Default: ARIA2USER@" << endl; + cout << " --ftp-type=TYPE Set FTP transfer type. TYPE is either 'binary'" << endl; + cout << " or 'ascii'." << endl; + cout << " Default: binary" << endl; + cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; + cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; + cout << " 'tunnel'." << endl; + cout << " -v, --version Print the version number and exit." << endl; + cout << " -h, --help Print this message and exit." << endl; cout << endl; cout << "URL:" << endl; cout << " You can specify multiple URLs. All URLs must point to the same file" << endl; @@ -166,6 +171,8 @@ int main(int argc, char* argv[]) { Option* op = new Option(); op->put(PREF_RETRY_WAIT, "5"); op->put(PREF_TIMEOUT, "60"); + op->put(PREF_MIN_SEGMENT_SIZE, "1048576");// 1M + op->put(PREF_MAX_TRIES, "5"); op->put(PREF_FTP_USER, "anonymous"); op->put(PREF_FTP_PASSWD, "ARIA2USER@"); op->put(PREF_FTP_TYPE, V_BINARY); @@ -182,6 +189,7 @@ int main(int argc, char* argv[]) { { "log", required_argument, NULL, 'l' }, { "split", required_argument, NULL, 's' }, { "timeout", required_argument, NULL, 't' }, + { "max-retries", required_argument, NULL, 'm' }, { "http-proxy", required_argument, &lopt, 1 }, { "http-user", required_argument, &lopt, 2 }, { "http-passwd", required_argument, &lopt, 3 }, @@ -195,11 +203,12 @@ int main(int argc, char* argv[]) { { "ftp-type", required_argument, &lopt, 11 }, { "ftp-pasv", no_argument, NULL, 'p' }, { "ftp-via-http-proxy", required_argument, &lopt, 12 }, + { "min-segment-size", required_argument, &lopt, 13 }, { "version", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "Dd:o:l:s:pt:vh", longOpts, &optIndex); + c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vh", longOpts, &optIndex); if(c == -1) { break; } @@ -278,6 +287,26 @@ int main(int argc, char* argv[]) { exit(1); } break; + case 13: { + string::size_type p = string(optarg).find_first_of("KM"); + int mult = 1; + if(p != string::npos) { + if(optarg[p] == 'K') { + mult = 1024; + } else if(optarg[p] == 'M') { + mult = 1024*1024; + } + optarg[p] = '\0'; + } + long long int size = strtoll(optarg, NULL, 10)*mult; + if(size <= 0) { + cerr << "min-segment-size invalid" << endl; + showUsage(); + exit(1); + } + op->put(PREF_MIN_SEGMENT_SIZE, Util::llitos(size)); + break; + } } break; } @@ -316,6 +345,16 @@ int main(int argc, char* argv[]) { } break; } + case 'm': { + int retries = (int)strtol(optarg, NULL, 10); + if(retries < 0) { + cerr << "max-retires invalid" << endl; + showUsage(); + exit(1); + } + op->put(PREF_MAX_TRIES, Util::itos(retries)); + break; + } case 'p': op->put(PREF_FTP_PASV_ENABLED, V_TRUE); break; @@ -363,6 +402,7 @@ int main(int argc, char* argv[]) { e->segmentMan->dir = dir; e->segmentMan->ufilename = ufilename; e->segmentMan->logger = logger; + e->segmentMan->option = op; vector requests; if(split > 0) { for(int i = 1; i <= split; i++) { diff --git a/src/prefs.h b/src/prefs.h index 847990da..bce670e4 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -36,7 +36,9 @@ // values: 1*digit #define PREF_TIMEOUT "timeout" // values: 1*digit -#define PREF_MAX_TRY "max_try" +#define PREF_MAX_TRIES "max_try" +// values: 1*digit +#define PREF_MIN_SEGMENT_SIZE "min_segment_size" /** * FTP related preferences