Magic-Mirror/aria2
1
0
Fork 0
mirror of https://github.com/aria2/aria2.git synced 2025-02-13 02:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

2008-04-13 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Browse Source 
Implemented auto protocol detection.
	Now you can do:
	aria2c -Z http://host/file file1.torrent file2.metalink
	(Note: -Z option is required for auto protcol detection.)
	Then aria2c downloads 3 files simultaneously:
	1. http://host/file
	2. file1.torrent <-- read local torrent file
	3. file2.metalink <-- read local Metalink file.

	Same thing goes with -i option. Assume your uris.txt contans:
	http://host/file
	file1.torrent
	file2.metalink
	Then you can do: aria2c -i uris.txt
	(Note: -Z option is not needed if -i option is given.)
	
	* src/main.cc
	* src/ProtocolDetector.{cc, h}
	* test/ProtocolDetectorTest.cc
This commit is contained in:
Tatsuhiro Tsujikawa 2008-04-13 13:30:12 +00:00
parent 0516a07fa6
commit 846e7c70f5
7 changed files with 312 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ChangeLog
View File

@ -1,3 +1,25 @@
2008-04-13 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Implemented auto protocol detection.
Now you can do:
aria2c -Z http://host/file file1.torrent file2.metalink
(Note: -Z option is required for auto protcol detection.)
Then aria2c downloads 3 files simultaneously:
1. http://host/file
2. file1.torrent <-- read local torrent file
3. file2.metalink <-- read local Metalink file.
Same thing goes with -i option. Assume your uris.txt contans:
http://host/file
file1.torrent
file2.metalink
Then you can do: aria2c -i uris.txt
(Note: -Z option is not needed if -i option is given.)
* src/main.cc
* src/ProtocolDetector.{cc, h}
* test/ProtocolDetectorTest.cc
2008-04-13 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Fixed compile error without gnutls/libgcrypt/libgpg-error and openSSL

3
src/Makefile.am
View File

@ -180,7 +180,8 @@ SRCS = Socket.h\
array_fun.h\
help_tags.h\
prefs.h\
usage_text.h
usage_text.h\
ProtocolDetector.cc ProtocolDetector.h
if ENABLE_MESSAGE_DIGEST
SRCS += IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\

18
src/Makefile.in
View File

@ -403,7 +403,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \
TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \
a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \
usage_text.h IteratableChunkChecksumValidator.cc \
usage_text.h ProtocolDetector.cc ProtocolDetector.h \
IteratableChunkChecksumValidator.cc \
IteratableChunkChecksumValidator.h \
IteratableChecksumValidator.cc IteratableChecksumValidator.h \
CheckIntegrityCommand.cc CheckIntegrityCommand.h \
@ -778,11 +779,12 @@ am__objects_14 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
ByteArrayDiskWriterFactory.$(OBJEXT) ServerHost.$(OBJEXT) \
HelpItem.$(OBJEXT) TaggedItem.$(OBJEXT) TagContainer.$(OBJEXT) \
HelpItemFactory.$(OBJEXT) SingleFileDownloadContext.$(OBJEXT) \
TimedHaltCommand.$(OBJEXT) $(am__objects_1) $(am__objects_2) \
$(am__objects_3) $(am__objects_4) $(am__objects_5) \
$(am__objects_6) $(am__objects_7) $(am__objects_8) \
$(am__objects_9) $(am__objects_10) $(am__objects_11) \
$(am__objects_12) $(am__objects_13)
TimedHaltCommand.$(OBJEXT) ProtocolDetector.$(OBJEXT) \
$(am__objects_1) $(am__objects_2) $(am__objects_3) \
$(am__objects_4) $(am__objects_5) $(am__objects_6) \
$(am__objects_7) $(am__objects_8) $(am__objects_9) \
$(am__objects_10) $(am__objects_11) $(am__objects_12) \
$(am__objects_13)
am_libaria2c_a_OBJECTS = $(am__objects_14)
libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
am__installdirs = "$(DESTDIR)$(bindir)"
@ -1116,7 +1118,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \
TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \
a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \
usage_text.h $(am__append_1) $(am__append_2) $(am__append_3) \
usage_text.h ProtocolDetector.cc ProtocolDetector.h \
$(am__append_1) $(am__append_2) $(am__append_3) \
$(am__append_4) $(am__append_5) $(am__append_6) \
$(am__append_7) $(am__append_8) $(am__append_9) \
$(am__append_10) $(am__append_11) $(am__append_12) \
@ -1431,6 +1434,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecedSegment.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecesMetalinkParserState.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ProtocolDetector.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RealtimeCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReceiverMSEHandshakeCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@

76
src/ProtocolDetector.cc Normal file
View File

@ -0,0 +1,76 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "ProtocolDetector.h"
#include "Request.h"
#include <cstring>
#include <fstream>
#include <iomanip>
namespace aria2 {
ProtocolDetector::ProtocolDetector() {}
ProtocolDetector::~ProtocolDetector() {}
bool ProtocolDetector::isStreamProtocol(const std::string& uri) const
{
return Request().setUrl(uri);
}
bool ProtocolDetector::guessTorrentFile(const std::string& uri) const
{
std::ifstream in(uri.c_str());
if(in) {
char head;
in >> head;
return head == 'd';
} else {
return false;
}
}
bool ProtocolDetector::guessMetalinkFile(const std::string& uri) const
{
std::ifstream in(uri.c_str());
if(in) {
char head[6];
in >> std::setw(6) >> head;
return strcmp(head, "<?xml") == 0;
} else {
return false;
}
}
} // namespace aria2

63
src/ProtocolDetector.h Normal file
View File

@ -0,0 +1,63 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef _D_PROTOCOL_DETECTOR_H_
#define _D_PROTOCOL_DETECTOR_H_
#include "common.h"
#include <string>
namespace aria2 {
class ProtocolDetector {
public:
ProtocolDetector();
~ProtocolDetector();
// Returns true if uri is http(s)/ftp, otherwise returns false.
bool isStreamProtocol(const std::string& uri) const;
// Returns true if ProtocolDetector thinks uri is a path of BitTorrent
// metainfo file, otherwise returns false.
bool guessTorrentFile(const std::string& uri) const;
// Returns true if ProtocolDetector thinks uri is a path of Metalink XML
// file, otherwise return false.
bool guessMetalinkFile(const std::string& uri) const;
};
} // namespace aria2
#endif // _D_PROTOCOL_DETECTOR_H_

109
src/main.cc
View File

@ -61,6 +61,7 @@
#include "DefaultBtContext.h"
#include "FileEntry.h"
#include "RequestGroup.h"
#include "ProtocolDetector.h"
#ifdef ENABLE_METALINK
# include "MetalinkHelper.h"
# include "Metalink2RequestGroup.h"
@ -74,6 +75,7 @@
#include <unistd.h>
#include <fstream>
#include <iostream>
#include <numeric>
extern char* optarg;
extern int optind, opterr, optopt;
#include <getopt.h>
@ -120,6 +122,24 @@ RequestGroupHandle createRequestGroup(const Option* op, const std::deque<std::st
extern Option* option_processing(int argc, char* const argv[]);
#ifdef ENABLE_BITTORRENT
SharedHandle<RequestGroup>
createBtRequestGroup(const std::string& torrentFilePath,
Option* op,
const std::deque<std::string>& auxUris)
{
SharedHandle<RequestGroup> rg = new RequestGroup(op, auxUris);
SharedHandle<DefaultBtContext> btContext = new DefaultBtContext();
btContext->load(torrentFilePath);// may throw exception
if(op->defined(PREF_PEER_ID_PREFIX)) {
btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
}
btContext->setDir(op->get(PREF_DIR));
rg->setDownloadContext(btContext);
btContext->setOwnerRequestGroup(rg.get());
return rg;
}
int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uri)
{
std::deque<std::string> nargs;
@ -132,15 +152,8 @@ int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uri)
ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT),
std::back_inserter(xargs));
RequestGroupHandle rg = new RequestGroup(op, xargs);
DefaultBtContextHandle btContext = new DefaultBtContext();
btContext->load(op->get(PREF_TORRENT_FILE));
if(op->defined(PREF_PEER_ID_PREFIX)) {
btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
}
btContext->setDir(op->get(PREF_DIR));
rg->setDownloadContext(btContext);
btContext->setOwnerRequestGroup(rg.get());
RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
op, xargs);
RequestGroups groups;
groups.push_back(rg);
@ -159,6 +172,55 @@ int32_t downloadMetalink(Option* op)
}
#endif // ENABLE_METALINK
class AccRequestGroup {
private:
ProtocolDetector _detector;
Option* _op;
public:
AccRequestGroup(Option* op):_op(op) {}
std::deque<SharedHandle<RequestGroup> >&
operator()(std::deque<SharedHandle<RequestGroup> >& groups,
const std::string& uri)
{
if(_detector.isStreamProtocol(uri)) {
std::deque<std::string> xuris;
for(size_t count = _op->getAsInt(PREF_SPLIT); count; --count) {
xuris.push_back(uri);
}
RequestGroupHandle rg = createRequestGroup(_op, xuris);
groups.push_back(rg);
}
#ifdef ENABLE_BITTORRENT
else if(_detector.guessTorrentFile(uri)) {
try {
groups.push_back(createBtRequestGroup(uri, _op,
std::deque<std::string>()));
} catch(RecoverableException* e) {
// error occurred while parsing torrent file.
// We simply ignore it.
}
}
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
else if(_detector.guessMetalinkFile(uri)) {
try {
std::deque<SharedHandle<RequestGroup> > metalinkGroups =
Metalink2RequestGroup(_op).generate(uri);
groups.insert(groups.end(), metalinkGroups.begin(), metalinkGroups.end());
} catch(RecoverableException* e) {
// error occurred while parsing metalink file.
// We simply ignore it.
}
}
#endif // ENABLE_METALINK
else {
LogFactory::getInstance()->error(MSG_UNRECOGNIZED_URI, (uri).c_str());
}
return groups;
}
};
int32_t downloadUriList(Option* op, std::istream& in)
{
UriListParser p;
@ -167,13 +229,17 @@ int32_t downloadUriList(Option* op, std::istream& in)
std::deque<std::string> uris = p.parseNext(in);
if(uris.size() == 1 && op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
std::deque<std::string> unfoldedURIs = unfoldURI(uris);
for(std::deque<std::string>::const_iterator itr = unfoldedURIs.begin();
itr != unfoldedURIs.end(); ++itr) {
std::deque<std::string> xuris;
ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT), std::back_inserter(xuris));
SharedHandle<RequestGroup> rg = createRequestGroup(op, xuris);
groups.push_back(rg);
}
std::deque<SharedHandle<RequestGroup> > thisGroups =
std::accumulate(unfoldedURIs.begin(), unfoldedURIs.end(),
std::deque<SharedHandle<RequestGroup> >(),
AccRequestGroup(op));
groups.insert(groups.end(), thisGroups.begin(), thisGroups.end());
} else if(uris.size() == 1) {
std::deque<SharedHandle<RequestGroup> > thisGroups =
std::accumulate(uris.begin(), uris.end(),
std::deque<SharedHandle<RequestGroup> >(),
AccRequestGroup(op));
groups.insert(groups.end(), thisGroups.begin(), thisGroups.end());
} else if(uris.size() > 0) {
std::deque<std::string> xuris;
ncopy(uris.begin(), uris.end(), op->getAsInt(PREF_SPLIT),
@ -208,14 +274,9 @@ int32_t downloadUri(Option* op, const std::deque<std::string>& uris)
}
RequestGroups groups;
if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) {
for(std::deque<std::string>::const_iterator itr = nargs.begin();
itr != nargs.end(); ++itr) {
std::deque<std::string> xuris;
ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT),
std::back_inserter(xuris));
RequestGroupHandle rg = createRequestGroup(op, xuris);
groups.push_back(rg);
}
groups = std::accumulate(nargs.begin(), nargs.end(),
std::deque<SharedHandle<RequestGroup> >(),
AccRequestGroup(op));
} else {
std::deque<std::string> xargs;
ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT),

53
test/ProtocolDetectorTest.cc Normal file
View File

@ -0,0 +1,53 @@
#include "ProtocolDetector.h"
#include "Exception.h"
#include "Util.h"
#include <iostream>
#include <cppunit/extensions/HelperMacros.h>
namespace aria2 {
class ProtocolDetectorTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(ProtocolDetectorTest);
CPPUNIT_TEST(testIsStreamProtocol);
CPPUNIT_TEST(testGuessTorrentFile);
CPPUNIT_TEST(testGuessMetalinkFile);
CPPUNIT_TEST_SUITE_END();
public:
void setUp() {}
void tearDown() {}
void testIsStreamProtocol();
void testGuessTorrentFile();
void testGuessMetalinkFile();
};
CPPUNIT_TEST_SUITE_REGISTRATION(ProtocolDetectorTest);
void ProtocolDetectorTest::testIsStreamProtocol()
{
ProtocolDetector detector;
CPPUNIT_ASSERT(detector.isStreamProtocol("http://localhost/index.html"));
CPPUNIT_ASSERT(detector.isStreamProtocol("ftp://localhost/index.html"));
CPPUNIT_ASSERT(!detector.isStreamProtocol("/home/web/localhost/index.html"));
}
void ProtocolDetectorTest::testGuessTorrentFile()
{
ProtocolDetector detector;
CPPUNIT_ASSERT(detector.guessTorrentFile("test.torrent"));
CPPUNIT_ASSERT(!detector.guessTorrentFile("http://localhost/test.torrent"));
CPPUNIT_ASSERT(!detector.guessTorrentFile("test.xml"));
}
void ProtocolDetectorTest::testGuessMetalinkFile()
{
ProtocolDetector detector;
CPPUNIT_ASSERT(detector.guessMetalinkFile("test.xml"));
CPPUNIT_ASSERT(!detector.guessMetalinkFile("http://localhost/test.xml"));
CPPUNIT_ASSERT(!detector.guessMetalinkFile("test.torrent"));
}
} // namespace aria2