diff --git a/ChangeLog b/ChangeLog index fa54a2e8..0247006f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +2008-04-13 Tatsuhiro Tsujikawa + + Implemented auto protocol detection. + Now you can do: + aria2c -Z http://host/file file1.torrent file2.metalink + (Note: -Z option is required for auto protcol detection.) + Then aria2c downloads 3 files simultaneously: + 1. http://host/file + 2. file1.torrent <-- read local torrent file + 3. file2.metalink <-- read local Metalink file. + + Same thing goes with -i option. Assume your uris.txt contans: + http://host/file + file1.torrent + file2.metalink + Then you can do: aria2c -i uris.txt + (Note: -Z option is not needed if -i option is given.) + + * src/main.cc + * src/ProtocolDetector.{cc, h} + * test/ProtocolDetectorTest.cc + 2008-04-13 Tatsuhiro Tsujikawa Fixed compile error without gnutls/libgcrypt/libgpg-error and openSSL diff --git a/src/Makefile.am b/src/Makefile.am index 3d549779..5c025d7e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -180,7 +180,8 @@ SRCS = Socket.h\ array_fun.h\ help_tags.h\ prefs.h\ - usage_text.h + usage_text.h\ + ProtocolDetector.cc ProtocolDetector.h if ENABLE_MESSAGE_DIGEST SRCS += IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 2e6c9861..330afe31 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -403,7 +403,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \ TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \ a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \ - usage_text.h IteratableChunkChecksumValidator.cc \ + usage_text.h ProtocolDetector.cc ProtocolDetector.h \ + IteratableChunkChecksumValidator.cc \ IteratableChunkChecksumValidator.h \ IteratableChecksumValidator.cc IteratableChecksumValidator.h \ CheckIntegrityCommand.cc CheckIntegrityCommand.h \ @@ -778,11 +779,12 @@ am__objects_14 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ ByteArrayDiskWriterFactory.$(OBJEXT) ServerHost.$(OBJEXT) \ HelpItem.$(OBJEXT) TaggedItem.$(OBJEXT) TagContainer.$(OBJEXT) \ HelpItemFactory.$(OBJEXT) SingleFileDownloadContext.$(OBJEXT) \ - TimedHaltCommand.$(OBJEXT) $(am__objects_1) $(am__objects_2) \ - $(am__objects_3) $(am__objects_4) $(am__objects_5) \ - $(am__objects_6) $(am__objects_7) $(am__objects_8) \ - $(am__objects_9) $(am__objects_10) $(am__objects_11) \ - $(am__objects_12) $(am__objects_13) + TimedHaltCommand.$(OBJEXT) ProtocolDetector.$(OBJEXT) \ + $(am__objects_1) $(am__objects_2) $(am__objects_3) \ + $(am__objects_4) $(am__objects_5) $(am__objects_6) \ + $(am__objects_7) $(am__objects_8) $(am__objects_9) \ + $(am__objects_10) $(am__objects_11) $(am__objects_12) \ + $(am__objects_13) am_libaria2c_a_OBJECTS = $(am__objects_14) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -1116,7 +1118,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \ TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \ a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \ - usage_text.h $(am__append_1) $(am__append_2) $(am__append_3) \ + usage_text.h ProtocolDetector.cc ProtocolDetector.h \ + $(am__append_1) $(am__append_2) $(am__append_3) \ $(am__append_4) $(am__append_5) $(am__append_6) \ $(am__append_7) $(am__append_8) $(am__append_9) \ $(am__append_10) $(am__append_11) $(am__append_12) \ @@ -1431,6 +1434,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecedSegment.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecesMetalinkParserState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ProtocolDetector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RealtimeCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReceiverMSEHandshakeCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@ diff --git a/src/ProtocolDetector.cc b/src/ProtocolDetector.cc new file mode 100644 index 00000000..9a01736f --- /dev/null +++ b/src/ProtocolDetector.cc @@ -0,0 +1,76 @@ +/* */ +#include "ProtocolDetector.h" +#include "Request.h" +#include +#include +#include + +namespace aria2 { + +ProtocolDetector::ProtocolDetector() {} + +ProtocolDetector::~ProtocolDetector() {} + +bool ProtocolDetector::isStreamProtocol(const std::string& uri) const +{ + return Request().setUrl(uri); +} + +bool ProtocolDetector::guessTorrentFile(const std::string& uri) const +{ + std::ifstream in(uri.c_str()); + if(in) { + char head; + in >> head; + return head == 'd'; + } else { + return false; + } +} + +bool ProtocolDetector::guessMetalinkFile(const std::string& uri) const +{ + std::ifstream in(uri.c_str()); + if(in) { + char head[6]; + in >> std::setw(6) >> head; + return strcmp(head, " */ +#ifndef _D_PROTOCOL_DETECTOR_H_ +#define _D_PROTOCOL_DETECTOR_H_ + +#include "common.h" +#include + +namespace aria2 { + +class ProtocolDetector { +public: + ProtocolDetector(); + + ~ProtocolDetector(); + + // Returns true if uri is http(s)/ftp, otherwise returns false. + bool isStreamProtocol(const std::string& uri) const; + + // Returns true if ProtocolDetector thinks uri is a path of BitTorrent + // metainfo file, otherwise returns false. + bool guessTorrentFile(const std::string& uri) const; + + // Returns true if ProtocolDetector thinks uri is a path of Metalink XML + // file, otherwise return false. + bool guessMetalinkFile(const std::string& uri) const; +}; + +} // namespace aria2 + +#endif // _D_PROTOCOL_DETECTOR_H_ diff --git a/src/main.cc b/src/main.cc index 51e71db2..8fe77b85 100644 --- a/src/main.cc +++ b/src/main.cc @@ -61,6 +61,7 @@ #include "DefaultBtContext.h" #include "FileEntry.h" #include "RequestGroup.h" +#include "ProtocolDetector.h" #ifdef ENABLE_METALINK # include "MetalinkHelper.h" # include "Metalink2RequestGroup.h" @@ -74,6 +75,7 @@ #include #include #include +#include extern char* optarg; extern int optind, opterr, optopt; #include @@ -120,6 +122,24 @@ RequestGroupHandle createRequestGroup(const Option* op, const std::deque +createBtRequestGroup(const std::string& torrentFilePath, + Option* op, + const std::deque& auxUris) +{ + SharedHandle rg = new RequestGroup(op, auxUris); + SharedHandle btContext = new DefaultBtContext(); + btContext->load(torrentFilePath);// may throw exception + if(op->defined(PREF_PEER_ID_PREFIX)) { + btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX)); + } + btContext->setDir(op->get(PREF_DIR)); + rg->setDownloadContext(btContext); + btContext->setOwnerRequestGroup(rg.get()); + return rg; +} + int32_t downloadBitTorrent(Option* op, const std::deque& uri) { std::deque nargs; @@ -132,15 +152,8 @@ int32_t downloadBitTorrent(Option* op, const std::deque& uri) ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT), std::back_inserter(xargs)); - RequestGroupHandle rg = new RequestGroup(op, xargs); - DefaultBtContextHandle btContext = new DefaultBtContext(); - btContext->load(op->get(PREF_TORRENT_FILE)); - if(op->defined(PREF_PEER_ID_PREFIX)) { - btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX)); - } - btContext->setDir(op->get(PREF_DIR)); - rg->setDownloadContext(btContext); - btContext->setOwnerRequestGroup(rg.get()); + RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE), + op, xargs); RequestGroups groups; groups.push_back(rg); @@ -159,6 +172,55 @@ int32_t downloadMetalink(Option* op) } #endif // ENABLE_METALINK +class AccRequestGroup { +private: + ProtocolDetector _detector; + Option* _op; +public: + AccRequestGroup(Option* op):_op(op) {} + + std::deque >& + operator()(std::deque >& groups, + const std::string& uri) + { + if(_detector.isStreamProtocol(uri)) { + std::deque xuris; + for(size_t count = _op->getAsInt(PREF_SPLIT); count; --count) { + xuris.push_back(uri); + } + RequestGroupHandle rg = createRequestGroup(_op, xuris); + groups.push_back(rg); + } +#ifdef ENABLE_BITTORRENT + else if(_detector.guessTorrentFile(uri)) { + try { + groups.push_back(createBtRequestGroup(uri, _op, + std::deque())); + } catch(RecoverableException* e) { + // error occurred while parsing torrent file. + // We simply ignore it. + } + } +#endif // ENABLE_BITTORRENT +#ifdef ENABLE_METALINK + else if(_detector.guessMetalinkFile(uri)) { + try { + std::deque > metalinkGroups = + Metalink2RequestGroup(_op).generate(uri); + groups.insert(groups.end(), metalinkGroups.begin(), metalinkGroups.end()); + } catch(RecoverableException* e) { + // error occurred while parsing metalink file. + // We simply ignore it. + } + } +#endif // ENABLE_METALINK + else { + LogFactory::getInstance()->error(MSG_UNRECOGNIZED_URI, (uri).c_str()); + } + return groups; + } +}; + int32_t downloadUriList(Option* op, std::istream& in) { UriListParser p; @@ -167,13 +229,17 @@ int32_t downloadUriList(Option* op, std::istream& in) std::deque uris = p.parseNext(in); if(uris.size() == 1 && op->get(PREF_PARAMETERIZED_URI) == V_TRUE) { std::deque unfoldedURIs = unfoldURI(uris); - for(std::deque::const_iterator itr = unfoldedURIs.begin(); - itr != unfoldedURIs.end(); ++itr) { - std::deque xuris; - ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT), std::back_inserter(xuris)); - SharedHandle rg = createRequestGroup(op, xuris); - groups.push_back(rg); - } + std::deque > thisGroups = + std::accumulate(unfoldedURIs.begin(), unfoldedURIs.end(), + std::deque >(), + AccRequestGroup(op)); + groups.insert(groups.end(), thisGroups.begin(), thisGroups.end()); + } else if(uris.size() == 1) { + std::deque > thisGroups = + std::accumulate(uris.begin(), uris.end(), + std::deque >(), + AccRequestGroup(op)); + groups.insert(groups.end(), thisGroups.begin(), thisGroups.end()); } else if(uris.size() > 0) { std::deque xuris; ncopy(uris.begin(), uris.end(), op->getAsInt(PREF_SPLIT), @@ -208,14 +274,9 @@ int32_t downloadUri(Option* op, const std::deque& uris) } RequestGroups groups; if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) { - for(std::deque::const_iterator itr = nargs.begin(); - itr != nargs.end(); ++itr) { - std::deque xuris; - ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT), - std::back_inserter(xuris)); - RequestGroupHandle rg = createRequestGroup(op, xuris); - groups.push_back(rg); - } + groups = std::accumulate(nargs.begin(), nargs.end(), + std::deque >(), + AccRequestGroup(op)); } else { std::deque xargs; ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT), diff --git a/test/ProtocolDetectorTest.cc b/test/ProtocolDetectorTest.cc new file mode 100644 index 00000000..e785bf83 --- /dev/null +++ b/test/ProtocolDetectorTest.cc @@ -0,0 +1,53 @@ +#include "ProtocolDetector.h" +#include "Exception.h" +#include "Util.h" +#include +#include + +namespace aria2 { + +class ProtocolDetectorTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(ProtocolDetectorTest); + CPPUNIT_TEST(testIsStreamProtocol); + CPPUNIT_TEST(testGuessTorrentFile); + CPPUNIT_TEST(testGuessMetalinkFile); + CPPUNIT_TEST_SUITE_END(); +public: + void setUp() {} + + void tearDown() {} + + void testIsStreamProtocol(); + void testGuessTorrentFile(); + void testGuessMetalinkFile(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION(ProtocolDetectorTest); + +void ProtocolDetectorTest::testIsStreamProtocol() +{ + ProtocolDetector detector; + CPPUNIT_ASSERT(detector.isStreamProtocol("http://localhost/index.html")); + CPPUNIT_ASSERT(detector.isStreamProtocol("ftp://localhost/index.html")); + CPPUNIT_ASSERT(!detector.isStreamProtocol("/home/web/localhost/index.html")); +} + +void ProtocolDetectorTest::testGuessTorrentFile() +{ + ProtocolDetector detector; + CPPUNIT_ASSERT(detector.guessTorrentFile("test.torrent")); + CPPUNIT_ASSERT(!detector.guessTorrentFile("http://localhost/test.torrent")); + CPPUNIT_ASSERT(!detector.guessTorrentFile("test.xml")); +} + +void ProtocolDetectorTest::testGuessMetalinkFile() +{ + ProtocolDetector detector; + CPPUNIT_ASSERT(detector.guessMetalinkFile("test.xml")); + CPPUNIT_ASSERT(!detector.guessMetalinkFile("http://localhost/test.xml")); + CPPUNIT_ASSERT(!detector.guessMetalinkFile("test.torrent")); +} + +} // namespace aria2