JDownloader2/jdownloader
1
0
Fork 0
mirror of https://github.com/mirror/jdownloader.git synced 2024-11-23 12:09:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

New Linkgrabber V2 API

Browse Source 
- refactoring - works like DownloadsListAPIV2 now.
 - Added Variants Support

git-svn-id: svn://svn.jdownloader.org/jdownloader/trunk@23929 ebf7c1c2-ba36-0410-9fe8-c592906822b4
This commit is contained in:
coalado 2014-01-21 16:19:57 +00:00
parent 5f701c0f3e
commit a0c50f3479
45 changed files with 1051 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
ressourcen/nsis/webinstaller_jd2_wa.nsi Normal file
View File

@ -0,0 +1,98 @@
Name "JDownloader"
OutFile ".\..\..\dist\WebInstallerJD2_c.exe"
CRCCheck on
XPStyle on
SetCompressor zlib #Don't use lzma here as filesize doesn't matter as long as it's <1MB
!define COMPANY "AppWork GmbH"
!define URL http://www.jdownloader.org
!define APPNAME "JDownloader"
!define VERSION 1.0.0.1
!include "timestamp.nsh"
!include "MUI.nsh"
!include "x64.nsh"
!define MUI_ICON .\res\install.ico
!insertmacro MUI_PAGE_INSTFILES
!AddPluginDir plugins
!include "LogicLib.nsh"
# Installer languages
!addincludedir ".\languages\"
!insertmacro MUI_LANGUAGE English
!include "en.webinstaller.nsh"
!insertmacro MUI_LANGUAGE German
!include "de.webinstaller.nsh"
VIAddVersionKey /LANG=${LANG_ENGLISH} ProductName "${APPNAME}"
VIAddVersionKey /LANG=${LANG_ENGLISH} CompanyName "${COMPANY}"
VIAddVersionKey /LANG=${LANG_ENGLISH} CompanyWebsite "${URL}"
VIAddVersionKey /LANG=${LANG_ENGLISH} FileDescription "${APPNAME} Setup for Windows"
VIAddVersionKey /LANG=${LANG_ENGLISH} LegalCopyright "${COMPANY}"
VIProductVersion "${VERSION}"
VIAddVersionKey /LANG=${LANG_ENGLISH} FileVersion "${VERSION}"
VIAddVersionKey /LANG=${LANG_ENGLISH} ProductVersion "${VERSION}"
Section
StrCpy $0 $HWNDPARENT
;System::Call "user32::ShowWindow(i r0, i 0)"
#http://nsis.sourceforge.net/Inetc_plug-in
IntOp $2 0 + 0 #count
#This might not be the reference implementation for a random number,
#but it's working and it's working good.
System::Call kernel32::GetTickCount()i.r3
IntOp $3 $3 % 4
${DoWhile} $2 < 3
${TimeStamp} $9
${If} ${RunningX64}
inetc::get /caption $(DownloadCaption) /useragent "JDownloaderWebSetup_inetc__jd2" /popup "JDownloaderSetup.exe" /translate $(inetc_url) $(inetc_downloading) $(inetc_connecting) $(inetc_file_name) $(inetc_received) $(inetc_file_size) $(inetc_remaining_time) $(inetc_total_time) "http://installer.jdownloader.org/$9/wi2/windows/64/jdownloader2_c" "$TEMP\JDownloaderSetup.exe"
${Else}
inetc::get /caption $(DownloadCaption) /useragent "JDownloaderWebSetup_inetc__jd2" /popup "JDownloaderSetup.exe" /translate $(inetc_url) $(inetc_downloading) $(inetc_connecting) $(inetc_file_name) $(inetc_received) $(inetc_file_size) $(inetc_remaining_time) $(inetc_total_time) "http://installer.jdownloader.org/$9/wi2/windows/32/jdownloader2_c" "$TEMP\JDownloaderSetup.exe"
${EndIf}
Pop $1
${If} $1 == "OK"
Exec '"$TEMP\JDownloaderSetup.exe"'
Delete /REBOOTOK "$TEMP\JDownloaderSetup.exe" #Won't be deleted immediately (executed before)
Quit
${ElseIf} $1 == "Cancelled"
Quit
${EndIf}
IntOp $2 $2 + 1 #count++;
IntOp $3 0 + 1 #current++;
${If} $3 > 3 #current = 0 if current > 3
IntOp $3 0 + 0
${EndIf}
${Loop}
MessageBox MB_ICONEXCLAMATION|MB_OK $(WebInstallFailed)
${TimeStamp} $9
${If} ${RunningX64}
ExecShell open "http://installer.jdownloader.org/$9/man/windows/64/jdownloader2_c" SW_SHOWMAXIMIZED
${Else}
ExecShell open "http://installer.jdownloader.org/$9/man/windows/32/jdownloader2_c" SW_SHOWMAXIMIZED
${EndIf}
Quit
SectionEnd

5
src/jd/controlling/ClipboardMonitoring.java
View File

@ -22,6 +22,7 @@ import java.util.StringTokenizer;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier; import jd.controlling.linkcrawler.CrawledLinkModifier;
@ -253,7 +254,7 @@ public class ClipboardMonitoring {
if (!StringUtils.isEmpty(handleThisRound)) { if (!StringUtils.isEmpty(handleThisRound)) {
if (FIRSTROUNDDONE) { if (FIRSTROUNDDONE) {
waitTimeout = minWaitTimeout; waitTimeout = minWaitTimeout;
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.CLIPBOARD, handleThisRound); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.CLIPBOARD, null), handleThisRound);
final HashSet<String> pws = PasswordUtils.getPasswords(handleThisRound); final HashSet<String> pws = PasswordUtils.getPasswords(handleThisRound);
if (pws != null && pws.size() > 0) { if (pws != null && pws.size() > 0) {
job.setCrawledLinkModifier(new CrawledLinkModifier() { job.setCrawledLinkModifier(new CrawledLinkModifier() {
@ -481,7 +482,7 @@ public class ClipboardMonitoring {
} }
String content = sb.toString(); String content = sb.toString();
if (!StringUtils.isEmpty(content)) { if (!StringUtils.isEmpty(content)) {
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.CLIPBOARD, content); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.CLIPBOARD, null), content);
job.setCustomSourceUrl(browserUrl); job.setCustomSourceUrl(browserUrl);
final HashSet<String> pws = PasswordUtils.getPasswords(content); final HashSet<String> pws = PasswordUtils.getPasswords(content);
if (pws != null && pws.size() > 0) { if (pws != null && pws.size() > 0) {

3
src/jd/controlling/downloadcontroller/DownloadWatchDog.java
View File

@ -52,6 +52,7 @@ import jd.controlling.downloadcontroller.event.DownloadWatchdogListener;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.packagecontroller.AbstractNode; import jd.controlling.packagecontroller.AbstractNode;
import jd.controlling.packagecontroller.AbstractPackageChildrenNodeFilter; import jd.controlling.packagecontroller.AbstractPackageChildrenNodeFilter;
import jd.controlling.proxy.ProxyController; import jd.controlling.proxy.ProxyController;
@ -2871,7 +2872,7 @@ public class DownloadWatchDog implements DownloadControllerListener, StateMachin
sb.append("file://"); sb.append("file://");
sb.append(file.getPath()); sb.append(file.getPath());
} }
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.DOWNLOADED_CONTAINER, sb.toString())); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.DOWNLOADED_CONTAINER, null), sb.toString()));
} }
}.start(); }.start();

65
src/jd/controlling/linkcollector/CrawledLinkStorable.java
View File

@ -57,15 +57,68 @@ public class CrawledLinkStorable implements Storable {
public String getOrigin() { public String getOrigin() {
if (link.getOrigin() == null) return null; if (link.getOrigin() == null) return null;
return link.getOrigin().name(); return link.getOrigin().getOrigin().name();
} }
public void setOrigin(String str) { public static class LinkOriginStorable implements Storable {
try { public LinkOriginStorable(/* Storable */) {
link.setOrigin(LinkOrigin.valueOf(str));
} catch (Throwable e) {
link.setOrigin(null);
} }
public LinkOriginStorable(LinkOriginDetails origin) {
this.id = origin.getOrigin().name();
this.details = origin.getDetails();
}
private String id;
private String details;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getDetails() {
return details;
}
public void setDetails(String details) {
this.details = details;
}
}
public LinkOriginStorable getOriginDetails() {
LinkOriginDetails origin = link.getOrigin();
if (origin == null) return null;
return new LinkOriginStorable(origin);
}
@Deprecated
public void setOrigin(String origin) {
if (origin != null) {
try {
LinkOrigin enu = LinkOrigin.valueOf(origin);
link.setOrigin(new LinkOriginDetails(enu, null));
} catch (Exception e) {
e.printStackTrace();
}
}
}
public void setOriginDetails(LinkOriginStorable origin) {
if (origin != null) {
try {
LinkOrigin enu = LinkOrigin.valueOf(origin.id);
link.setOrigin(new LinkOriginDetails(enu, origin.details));
} catch (Exception e) {
e.printStackTrace();
}
}
} }
public long getUID() { public long getUID() {

8
src/jd/controlling/linkcollector/LinkCollectingJob.java
View File

@ -32,11 +32,11 @@ public class LinkCollectingJob {
this.customSourceUrl = customSource; this.customSourceUrl = customSource;
} }
public LinkCollectingJob(LinkOrigin origin) { public LinkCollectingJob(LinkOriginDetails origin) {
this(origin, null); this(origin, null);
} }
public LinkCollectingJob(LinkOrigin origin, String jobContent) { public LinkCollectingJob(LinkOriginDetails origin, String jobContent) {
this.jobContent = jobContent; this.jobContent = jobContent;
this.origin = origin; this.origin = origin;
} }
@ -49,9 +49,9 @@ public class LinkCollectingJob {
this.jobContent = text; this.jobContent = text;
} }
private final LinkOrigin origin; private final LinkOriginDetails origin;
public LinkOrigin getOrigin() { public LinkOriginDetails getOrigin() {
return origin; return origin;
} }

104
src/jd/controlling/linkcollector/LinkCollector.java
View File

@ -15,6 +15,7 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.WeakHashMap;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
@ -26,7 +27,6 @@ import jd.controlling.linkchecker.LinkChecker;
import jd.controlling.linkchecker.LinkCheckerHandler; import jd.controlling.linkchecker.LinkCheckerHandler;
import jd.controlling.linkcrawler.CheckableLink; import jd.controlling.linkcrawler.CheckableLink;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.controlling.linkcrawler.CrawledLinkProperty; import jd.controlling.linkcrawler.CrawledLinkProperty;
import jd.controlling.linkcrawler.CrawledPackage; import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.CrawledPackage.TYPE; import jd.controlling.linkcrawler.CrawledPackage.TYPE;
@ -85,6 +85,7 @@ import org.jdownloader.gui.views.linkgrabber.LinkGrabberTableModel;
import org.jdownloader.gui.views.linkgrabber.LinkgrabberSearchField; import org.jdownloader.gui.views.linkgrabber.LinkgrabberSearchField;
import org.jdownloader.gui.views.linkgrabber.contextmenu.MenuManagerLinkgrabberTableContext; import org.jdownloader.gui.views.linkgrabber.contextmenu.MenuManagerLinkgrabberTableContext;
import org.jdownloader.images.NewTheme; import org.jdownloader.images.NewTheme;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
import org.jdownloader.plugins.controller.UpdateRequiredClassNotFoundException; import org.jdownloader.plugins.controller.UpdateRequiredClassNotFoundException;
import org.jdownloader.plugins.controller.host.HostPluginController; import org.jdownloader.plugins.controller.host.HostPluginController;
import org.jdownloader.plugins.controller.host.LazyHostPlugin; import org.jdownloader.plugins.controller.host.LazyHostPlugin;
@ -644,7 +645,7 @@ public class LinkCollector extends PackageController<CrawledPackage, CrawledLink
List<CrawledLink> add = new ArrayList<CrawledLink>(1); List<CrawledLink> add = new ArrayList<CrawledLink>(1);
add.add(link); add.add(link);
LinkCollector.this.moveOrAddAt(getPermanentOfflineCrawledPackage(), add, -1); LinkCollector.this.moveOrAddAt(getPermanentOfflineCrawledPackage(), add, -1);
} else if (!ignoreSpecialPackages && link.getLinkState() == LinkState.OFFLINE && org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.OFFLINE_PACKAGE_ENABLED.getValue()) { } else if (!ignoreSpecialPackages && link.getLinkState() == AvailableLinkState.OFFLINE && org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.OFFLINE_PACKAGE_ENABLED.getValue()) {
java.util.List<CrawledLink> list = getIdentifiedMap(identifier, offlineMap); java.util.List<CrawledLink> list = getIdentifiedMap(identifier, offlineMap);
list.add(link); list.add(link);
List<CrawledLink> add = new ArrayList<CrawledLink>(1); List<CrawledLink> add = new ArrayList<CrawledLink>(1);
@ -1809,4 +1810,103 @@ public class LinkCollector extends PackageController<CrawledPackage, CrawledLink
}); });
} }
private WeakHashMap<UniqueAlltimeID, CrawledLink> linkMap = new WeakHashMap<UniqueAlltimeID, CrawledLink>();
private WeakHashMap<UniqueAlltimeID, CrawledPackage> packageMap2 = new WeakHashMap<UniqueAlltimeID, CrawledPackage>();
public CrawledLink getLinkByID(long longID) {
UniqueAlltimeID linkId = new UniqueAlltimeID(longID);
synchronized (linkMap) {
CrawledLink ret = linkMap.get(linkId);
if (ret != null) return ret;
updateIdMaps();
return linkMap.get(linkId);
}
}
protected void updateIdMaps() {
LinkCollector dlc = LinkCollector.getInstance();
boolean readL = dlc.readLock();
try {
for (CrawledPackage pkg : dlc.getPackages()) {
packageMap2.put(pkg.getUniqueID(), pkg);
boolean readL2 = pkg.getModifyLock().readLock();
try {
for (CrawledLink child : pkg.getChildren()) {
linkMap.put(child.getUniqueID(), child);
}
} finally {
pkg.getModifyLock().readUnlock(readL2);
}
}
} finally {
dlc.readUnlock(readL);
}
}
public CrawledPackage getPackageByID(long longID) {
UniqueAlltimeID packageId = new UniqueAlltimeID(longID);
synchronized (linkMap) {
CrawledPackage ret = packageMap2.get(packageId);
if (ret != null) return ret;
updateIdMaps();
return packageMap2.get(packageId);
}
}
public List<CrawledLink> getLinksByID(long[] packageUUIDs) {
boolean updated = false;
synchronized (linkMap) {
ArrayList<CrawledLink> ret = new ArrayList<CrawledLink>();
for (long longID : packageUUIDs) {
UniqueAlltimeID l = new UniqueAlltimeID(longID);
CrawledLink p = linkMap.get(l);
if (p != null) {
ret.add(p);
} else {
// only update once
if (updated) continue;
updateIdMaps();
updated = true;
p = linkMap.get(l);
if (p != null) {
ret.add(p);
}
}
}
return ret;
}
}
public List<CrawledPackage> getPackagesByID(long[] packageUUIDs) {
boolean updated = false;
synchronized (linkMap) {
ArrayList<CrawledPackage> ret = new ArrayList<CrawledPackage>();
for (long longID : packageUUIDs) {
UniqueAlltimeID l = new UniqueAlltimeID(longID);
CrawledPackage p = packageMap2.get(l);
if (p != null) {
ret.add(p);
} else {
// only update once
if (updated) continue;
updateIdMaps();
updated = true;
p = packageMap2.get(l);
if (p != null) {
ret.add(p);
}
}
}
return ret;
}
}
} }

33
src/jd/controlling/linkcollector/LinkOriginDetails.java Normal file
View File

@ -0,0 +1,33 @@
package jd.controlling.linkcollector;
public class LinkOriginDetails {
/**
* @param origin
* @param details
*/
public LinkOriginDetails(LinkOrigin origin, String details) {
super();
this.origin = origin;
this.details = details;
}
private LinkOrigin origin;
public LinkOrigin getOrigin() {
return origin;
}
public void setOrigin(LinkOrigin origin) {
this.origin = origin;
}
public String getDetails() {
return details;
}
public void setDetails(String details) {
this.details = details;
}
private String details;
}

37
src/jd/controlling/linkcrawler/CrawledLink.java
View File

@ -7,7 +7,7 @@ import javax.swing.ImageIcon;
import jd.controlling.linkcollector.LinkCollectingInformation; import jd.controlling.linkcollector.LinkCollectingInformation;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.packagecontroller.AbstractNode; import jd.controlling.packagecontroller.AbstractNode;
import jd.controlling.packagecontroller.AbstractNodeNotifier; import jd.controlling.packagecontroller.AbstractNodeNotifier;
import jd.controlling.packagecontroller.AbstractPackageChildrenNode; import jd.controlling.packagecontroller.AbstractPackageChildrenNode;
@ -27,16 +27,10 @@ import org.jdownloader.controlling.UniqueAlltimeID;
import org.jdownloader.controlling.filter.FilterRule; import org.jdownloader.controlling.filter.FilterRule;
import org.jdownloader.controlling.packagizer.PackagizerController; import org.jdownloader.controlling.packagizer.PackagizerController;
import org.jdownloader.extensions.extraction.BooleanStatus; import org.jdownloader.extensions.extraction.BooleanStatus;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
public class CrawledLink implements AbstractPackageChildrenNode<CrawledPackage>, CheckableLink, AbstractNodeNotifier { public class CrawledLink implements AbstractPackageChildrenNode<CrawledPackage>, CheckableLink, AbstractNodeNotifier {
public static enum LinkState {
ONLINE,
OFFLINE,
UNKNOWN,
TEMP_UNKNOWN
}
private boolean crawlDeep = false; private boolean crawlDeep = false;
public boolean isCrawlDeep() { public boolean isCrawlDeep() {
@ -54,17 +48,17 @@ public class CrawledLink implements AbstractPackageChildrenNode<CrawledPackage>,
private boolean autoConfirmEnabled = false; private boolean autoConfirmEnabled = false;
private transient UniqueAlltimeID uniqueID = new UniqueAlltimeID(); private transient UniqueAlltimeID uniqueID = new UniqueAlltimeID();
private LinkOrigin origin; private LinkOriginDetails origin;
public boolean isAutoConfirmEnabled() { public boolean isAutoConfirmEnabled() {
return autoConfirmEnabled; return autoConfirmEnabled;
} }
public void setOrigin(LinkOrigin source) { public void setOrigin(LinkOriginDetails source) {
this.origin = source; this.origin = source;
} }
public LinkOrigin getOrigin() { public LinkOriginDetails getOrigin() {
return origin; return origin;
} }
@ -116,11 +110,12 @@ public class CrawledLink implements AbstractPackageChildrenNode<CrawledPackage>,
} }
/** /**
* Linkid should be unique for a certain link. in most cases, this is the url itself, but somtimes (youtube e.g.) the id contains info about how to prozess * Linkid should be unique for a certain link. in most cases, this is the url itself, but somtimes (youtube e.g.) the id contains info
* the file afterwards. * about how to prozess the file afterwards.
* *
* example:<br> * example:<br>
* 2 youtube links may have the same url, but the one will be converted into mp3, and the other stays flv. url is the same, but linkID different. * 2 youtube links may have the same url, but the one will be converted into mp3, and the other stays flv. url is the same, but linkID
* different.
* *
* @return * @return
*/ */
@ -394,22 +389,22 @@ public class CrawledLink implements AbstractPackageChildrenNode<CrawledPackage>,
return matchingFilter; return matchingFilter;
} }
public LinkState getLinkState() { public AvailableLinkState getLinkState() {
if (dlLink != null) { if (dlLink != null) {
switch (dlLink.getAvailableStatus()) { switch (dlLink.getAvailableStatus()) {
case FALSE: case FALSE:
return LinkState.OFFLINE; return AvailableLinkState.OFFLINE;
case TRUE: case TRUE:
return LinkState.ONLINE; return AvailableLinkState.ONLINE;
case UNCHECKABLE: case UNCHECKABLE:
return LinkState.TEMP_UNKNOWN; return AvailableLinkState.TEMP_UNKNOWN;
case UNCHECKED: case UNCHECKED:
return LinkState.UNKNOWN; return AvailableLinkState.UNKNOWN;
default: default:
return LinkState.UNKNOWN; return AvailableLinkState.UNKNOWN;
} }
} }
return LinkState.UNKNOWN; return AvailableLinkState.UNKNOWN;
} }
public Priority getPriority() { public Priority getPriority() {

6
src/jd/controlling/linkcrawler/CrawledPackageView.java
View File

@ -6,7 +6,6 @@ import java.util.List;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.controlling.packagecontroller.ChildrenView; import jd.controlling.packagecontroller.ChildrenView;
import jd.plugins.DownloadLink; import jd.plugins.DownloadLink;
@ -15,6 +14,7 @@ import org.jdownloader.DomainInfo;
import org.jdownloader.controlling.Priority; import org.jdownloader.controlling.Priority;
import org.jdownloader.gui.translate._GUI; import org.jdownloader.gui.translate._GUI;
import org.jdownloader.gui.views.downloads.columns.AvailabilityColumn; import org.jdownloader.gui.views.downloads.columns.AvailabilityColumn;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
public class CrawledPackageView extends ChildrenView<CrawledLink> { public class CrawledPackageView extends ChildrenView<CrawledLink> {
@ -139,10 +139,10 @@ public class CrawledPackageView extends ChildrenView<CrawledLink> {
// enabled // enabled
if (link.isEnabled()) tmp.newEnabled = true; if (link.isEnabled()) tmp.newEnabled = true;
if (link.getLinkState() == LinkState.OFFLINE) { if (link.getLinkState() == AvailableLinkState.OFFLINE) {
// offline // offline
tmp.newOffline++; tmp.newOffline++;
} else if (link.getLinkState() == LinkState.ONLINE) { } else if (link.getLinkState() == AvailableLinkState.ONLINE) {
// online // online
tmp.newOnline++; tmp.newOnline++;
} }

7
src/jd/gui/swing/MacOSApplicationAdapter.java
View File

@ -31,6 +31,7 @@ import jd.controlling.downloadcontroller.event.DownloadWatchdogListener;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.gui.swing.dialog.AboutDialog; import jd.gui.swing.dialog.AboutDialog;
import jd.gui.swing.jdgui.JDGui; import jd.gui.swing.jdgui.JDGui;
import jd.gui.swing.jdgui.views.settings.ConfigurationView; import jd.gui.swing.jdgui.views.settings.ConfigurationView;
@ -101,7 +102,7 @@ public class MacOSApplicationAdapter implements QuitHandler, AboutHandler, Prefe
} }
if (adapter.openURIlinks != null) { if (adapter.openURIlinks != null) {
LogController.GL.info("Distribute links: " + adapter.openURIlinks); LogController.GL.info("Distribute links: " + adapter.openURIlinks);
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.START_PARAMETER, adapter.openURIlinks)); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.START_PARAMETER, null), adapter.openURIlinks));
adapter.openURIlinks = null; adapter.openURIlinks = null;
} }
} }
@ -328,7 +329,7 @@ public class MacOSApplicationAdapter implements QuitHandler, AboutHandler, Prefe
sb.append("file://"); sb.append("file://");
sb.append(file.getPath()); sb.append(file.getPath());
} }
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.MAC_DOCK, sb.toString())); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.MAC_DOCK, null), sb.toString()));
} }
public void openURI(AppEvent.OpenURIEvent e) { public void openURI(AppEvent.OpenURIEvent e) {
@ -337,7 +338,7 @@ public class MacOSApplicationAdapter implements QuitHandler, AboutHandler, Prefe
String links = e.getURI().toString(); String links = e.getURI().toString();
if (SecondLevelLaunch.GUI_COMPLETE.isReached()) { if (SecondLevelLaunch.GUI_COMPLETE.isReached()) {
LogController.GL.info("Distribute links: " + links); LogController.GL.info("Distribute links: " + links);
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.MAC_DOCK, links)); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.MAC_DOCK, null), links));
} else { } else {
openURIlinks = links; openURIlinks = links;
} }

3
src/jd/gui/swing/jdgui/menu/actions/AddContainerAction.java
View File

@ -23,6 +23,7 @@ import java.io.File;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.gui.UserIO; import jd.gui.UserIO;
import jd.nutils.io.JDFileFilter; import jd.nutils.io.JDFileFilter;
@ -51,7 +52,7 @@ public class AddContainerAction extends CustomizableAppAction {
sb.append("file://"); sb.append("file://");
sb.append(r.getAbsolutePath()); sb.append(r.getAbsolutePath());
} }
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.ADD_CONTAINER_ACTION, sb.toString())); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.ADD_CONTAINER_ACTION, null), sb.toString()));
} }
} }

2
src/jd/plugins/decrypter/YoutubeVariantInterface.java
View File

@ -32,8 +32,6 @@ public interface YoutubeVariantInterface extends LinkVariant {
String getFileExtension(); String getFileExtension();
String getUniqueId();
String getMediaTypeID(); String getMediaTypeID();
YoutubeITAG getiTagVideo(); YoutubeITAG getiTagVideo();

2
src/org/jdownloader/api/RemoteAPIController.java
View File

@ -56,6 +56,7 @@ import org.jdownloader.api.jd.JDAPIImpl;
import org.jdownloader.api.jdanywhere.JDAnywhereAPI; import org.jdownloader.api.jdanywhere.JDAnywhereAPI;
import org.jdownloader.api.linkcollector.LinkCollectorAPIImpl; import org.jdownloader.api.linkcollector.LinkCollectorAPIImpl;
import org.jdownloader.api.linkcollector.LinkCollectorEventPublisher; import org.jdownloader.api.linkcollector.LinkCollectorEventPublisher;
import org.jdownloader.api.linkcollector.v2.LinkCollectorAPIImplV2;
import org.jdownloader.api.linkcrawler.LinkCrawlerAPIImpl; import org.jdownloader.api.linkcrawler.LinkCrawlerAPIImpl;
import org.jdownloader.api.linkcrawler.LinkCrawlerEventPublisher; import org.jdownloader.api.linkcrawler.LinkCrawlerEventPublisher;
import org.jdownloader.api.myjdownloader.MyJDownloaderPostRequest; import org.jdownloader.api.myjdownloader.MyJDownloaderPostRequest;
@ -231,6 +232,7 @@ public class RemoteAPIController {
register(new AccountAPIImpl()); register(new AccountAPIImpl());
register(new AccountAPIImplV2()); register(new AccountAPIImplV2());
register(new LinkCollectorAPIImpl()); register(new LinkCollectorAPIImpl());
register(new LinkCollectorAPIImplV2());
register(new ContentAPIImpl()); register(new ContentAPIImpl());
register(new ContentAPIImplV2()); register(new ContentAPIImplV2());
register(new PollingAPIImpl()); register(new PollingAPIImpl());

18
src/org/jdownloader/api/cnl2/ExternInterfaceImpl.java
View File

@ -14,6 +14,7 @@ import javax.swing.ImageIcon;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier; import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.LinkCrawler; import jd.controlling.linkcrawler.LinkCrawler;
@ -122,7 +123,7 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
String jk = HttpRequest.getParameterbyKey(request, "jk"); String jk = HttpRequest.getParameterbyKey(request, "jk");
String k = HttpRequest.getParameterbyKey(request, "k"); String k = HttpRequest.getParameterbyKey(request, "k");
String urls = decrypt(crypted, jk, k); String urls = decrypt(crypted, jk, k);
clickAndLoad2Add(urls, request); clickAndLoad2Add(new LinkOriginDetails(LinkOrigin.CNL, request.getRequestHeaders().getValue("user-agent")), urls, request);
/* /*
* we need the \r\n else the website will not handle response correctly * we need the \r\n else the website will not handle response correctly
*/ */
@ -135,16 +136,16 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
// For My JD API // For My JD API
public void addcrypted2Remote(String crypted, String jk, String source) { public void addcrypted2Remote(String crypted, String jk, String source) {
String urls = decrypt(crypted, jk, null); String urls = decrypt(crypted, jk, null);
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.CNL, urls); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.CNL, null), urls);
job.setCustomSourceUrl(source); job.setCustomSourceUrl(source);
LinkCollector.getInstance().addCrawlerJob(job); LinkCollector.getInstance().addCrawlerJob(job);
} }
private void clickAndLoad2Add(String urls, RemoteAPIRequest request) throws IOException { private void clickAndLoad2Add(LinkOriginDetails origin, String urls, RemoteAPIRequest request) throws IOException {
final String finalPasswords = HttpRequest.getParameterbyKey(request, "passwords"); final String finalPasswords = HttpRequest.getParameterbyKey(request, "passwords");
String source = HttpRequest.getParameterbyKey(request, "source"); String source = HttpRequest.getParameterbyKey(request, "source");
final String finalComment = HttpRequest.getParameterbyKey(request, "comment"); final String finalComment = HttpRequest.getParameterbyKey(request, "comment");
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.CNL, urls); LinkCollectingJob job = new LinkCollectingJob(origin, urls);
final String finalDestination = HttpRequest.getParameterbyKey(request, "dir"); final String finalDestination = HttpRequest.getParameterbyKey(request, "dir");
job.setCustomSourceUrl(source); job.setCustomSourceUrl(source);
final String finalPackageName = HttpRequest.getParameterbyKey(request, "package"); final String finalPackageName = HttpRequest.getParameterbyKey(request, "package");
@ -245,7 +246,7 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
try { try {
askPermission(request); askPermission(request);
String urls = HttpRequest.getParameterbyKey(request, "urls"); String urls = HttpRequest.getParameterbyKey(request, "urls");
clickAndLoad2Add(urls, request); clickAndLoad2Add(new LinkOriginDetails(LinkOrigin.CNL, request.getRequestHeaders().getValue("user-agent")), urls, request);
writeString(response, request, "success\r\n", true); writeString(response, request, "success\r\n", true);
} catch (Throwable e) { } catch (Throwable e) {
writeString(response, request, "failed " + e.getMessage() + "\r\n", true); writeString(response, request, "failed " + e.getMessage() + "\r\n", true);
@ -255,7 +256,7 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
// For My JD API // For My JD API
@Override @Override
public void add(String passwords, String source, String urls) throws InternalApiException { public void add(String passwords, String source, String urls) throws InternalApiException {
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.CNL, urls); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.CNL, null), urls);
// String dir = HttpRequest.getParameterbyKey(request, "dir"); // String dir = HttpRequest.getParameterbyKey(request, "dir");
// if (!StringUtils.isEmpty(dir)) { // if (!StringUtils.isEmpty(dir)) {
// job.setOutputFolder(new File(dir)); // job.setOutputFolder(new File(dir));
@ -289,7 +290,7 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
File tmp = Application.getTempResource("jd_" + System.currentTimeMillis() + ".dlc"); File tmp = Application.getTempResource("jd_" + System.currentTimeMillis() + ".dlc");
IO.writeToFile(tmp, dlc.getBytes("UTF-8")); IO.writeToFile(tmp, dlc.getBytes("UTF-8"));
String url = "file://" + tmp.getAbsolutePath(); String url = "file://" + tmp.getAbsolutePath();
clickAndLoad2Add(url, request); clickAndLoad2Add(new LinkOriginDetails(LinkOrigin.CNL, request.getRequestHeaders().getValue("user-agent")), url, request);
writeString(response, request, "success\r\n", true); writeString(response, request, "success\r\n", true);
} catch (Throwable e) { } catch (Throwable e) {
writeString(response, request, "failed " + e.getMessage() + "\r\n", true); writeString(response, request, "failed " + e.getMessage() + "\r\n", true);
@ -392,7 +393,7 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
/* /*
* create LinkCollectingJob to forward general Information like directory, autostart... * create LinkCollectingJob to forward general Information like directory, autostart...
*/ */
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.FLASHGOT, null); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.FLASHGOT, request.getRequestHeaders().getValue("user-agent")), null);
final String finalPackageName = HttpRequest.getParameterbyKey(request, "package"); final String finalPackageName = HttpRequest.getParameterbyKey(request, "package");
final String finalDestination = HttpRequest.getParameterbyKey(request, "dir"); final String finalDestination = HttpRequest.getParameterbyKey(request, "dir");
@ -505,4 +506,5 @@ public class ExternInterfaceImpl implements Cnl2APIBasics, Cnl2APIFlash {
throw new InternalApiException(e); throw new InternalApiException(e);
} }
} }
} }

1
src/org/jdownloader/api/downloads/DownloadsAPIImpl.java
View File

@ -38,6 +38,7 @@ public class DownloadsAPIImpl implements DownloadsAPI {
return DownloadWatchDog.getInstance().getStateMachine().getState().getLabel(); return DownloadWatchDog.getInstance().getStateMachine().getState().getLabel();
} }
@Deprecated
public DownloadsAPIImpl() { public DownloadsAPIImpl() {
} }

5
src/org/jdownloader/api/downloads/v2/DownloadsAPIV2.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import org.appwork.remoteapi.RemoteAPIInterface; import org.appwork.remoteapi.RemoteAPIInterface;
import org.appwork.remoteapi.annotations.ApiNamespace; import org.appwork.remoteapi.annotations.ApiNamespace;
import org.appwork.remoteapi.exceptions.BadParameterException;
@ApiNamespace("downloadsV2") @ApiNamespace("downloadsV2")
public interface DownloadsAPIV2 extends RemoteAPIInterface { public interface DownloadsAPIV2 extends RemoteAPIInterface {
@ -32,7 +33,7 @@ public interface DownloadsAPIV2 extends RemoteAPIInterface {
* @return * @return
*/ */
List<FilePackageAPIStorableV2> queryPackages(PackageQueryStorable queryParams); List<FilePackageAPIStorableV2> queryPackages(PackageQueryStorable queryParams) throws BadParameterException;
void removeLinks(final long[] linkIds, final long[] packageIds); void removeLinks(final long[] linkIds, final long[] packageIds);
@ -73,6 +74,6 @@ public interface DownloadsAPIV2 extends RemoteAPIInterface {
* *
* @return * @return
*/ */
List<DownloadLinkAPIStorableV2> queryLinks(LinkQueryStorable queryParams); List<DownloadLinkAPIStorableV2> queryLinks(LinkQueryStorable queryParams) throws BadParameterException;
} }

8
src/org/jdownloader/api/downloads/v2/DownloadsAPIV2Impl.java
View File

@ -12,6 +12,7 @@ import jd.plugins.FilePackage;
import jd.plugins.FilePackageView; import jd.plugins.FilePackageView;
import jd.plugins.PluginProgress; import jd.plugins.PluginProgress;
import org.appwork.remoteapi.exceptions.BadParameterException;
import org.jdownloader.DomainInfo; import org.jdownloader.DomainInfo;
import org.jdownloader.api.RemoteAPIController; import org.jdownloader.api.RemoteAPIController;
import org.jdownloader.gui.views.SelectionInfo; import org.jdownloader.gui.views.SelectionInfo;
@ -25,7 +26,8 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
} }
@Override @Override
public List<FilePackageAPIStorableV2> queryPackages(PackageQueryStorable queryParams) { public List<FilePackageAPIStorableV2> queryPackages(PackageQueryStorable queryParams) throws BadParameterException {
DownloadController dlc = DownloadController.getInstance(); DownloadController dlc = DownloadController.getInstance();
DownloadWatchDog dwd = DownloadWatchDog.getInstance(); DownloadWatchDog dwd = DownloadWatchDog.getInstance();
@ -65,7 +67,7 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
} }
if (queryParams.isChildCount()) { if (queryParams.isChildCount()) {
fps.setChildCount(fpView.getItems().size()); fps.setChildCount(fp.size());
} }
if (queryParams.isHosts()) { if (queryParams.isHosts()) {
DomainInfo[] di = fpView.getDomainInfos(); DomainInfo[] di = fpView.getDomainInfos();
@ -352,7 +354,7 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
return ret; return ret;
} }
protected static HashSet<Long> createLookupSet(long[] linkIds) { public static HashSet<Long> createLookupSet(long[] linkIds) {
if (linkIds == null || linkIds.length == 0) return null; if (linkIds == null || linkIds.length == 0) return null;
HashSet<Long> linkLookup = new HashSet<Long>(); HashSet<Long> linkLookup = new HashSet<Long>();
for (long l : linkIds) for (long l : linkIds)

1
src/org/jdownloader/api/downloads/v2/PackageQueryStorable.java
View File

@ -7,4 +7,5 @@ public class PackageQueryStorable extends PackageQuery implements Storable {
public PackageQueryStorable(/* Storable */) { public PackageQueryStorable(/* Storable */) {
} }
} }

3
src/org/jdownloader/api/jdanywhere/api/LinkCrawlerApi.java
View File

@ -7,6 +7,7 @@ import java.util.List;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledPackage; import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.packagecontroller.AbstractPackageChildrenNodeFilter; import jd.controlling.packagecontroller.AbstractPackageChildrenNodeFilter;
@ -116,7 +117,7 @@ public class LinkCrawlerApi implements ILinkCrawlerApi {
File tmp = Application.getTempResource("jd_" + System.currentTimeMillis() + ".dlc"); File tmp = Application.getTempResource("jd_" + System.currentTimeMillis() + ".dlc");
IO.writeToFile(tmp, dlc.getBytes("UTF-8")); IO.writeToFile(tmp, dlc.getBytes("UTF-8"));
String url = "file://" + tmp.getAbsolutePath(); String url = "file://" + tmp.getAbsolutePath();
LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.MYJD, url); LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.MYJD, null), url);
LinkCollector.getInstance().addCrawlerJob(job); LinkCollector.getInstance().addCrawlerJob(job);
} catch (Throwable e) { } catch (Throwable e) {
} }

11
src/org/jdownloader/api/linkcollector/LinkCollectorAPIImpl.java
View File

@ -1,7 +1,6 @@
package org.jdownloader.api.linkcollector; package org.jdownloader.api.linkcollector;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -9,8 +8,8 @@ import java.util.Set;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.controlling.linkcrawler.CrawledLinkModifier; import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.CrawledPackage; import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.CrawledPackageView; import jd.controlling.linkcrawler.CrawledPackageView;
@ -22,8 +21,10 @@ import org.appwork.remoteapi.APIQuery;
import org.appwork.utils.StringUtils; import org.appwork.utils.StringUtils;
import org.jdownloader.gui.packagehistorycontroller.DownloadPathHistoryManager; import org.jdownloader.gui.packagehistorycontroller.DownloadPathHistoryManager;
import org.jdownloader.gui.views.SelectionInfo; import org.jdownloader.gui.views.SelectionInfo;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
import org.jdownloader.settings.GeneralSettings; import org.jdownloader.settings.GeneralSettings;
@Deprecated
public class LinkCollectorAPIImpl implements LinkCollectorAPI { public class LinkCollectorAPIImpl implements LinkCollectorAPI {
@Override @Override
@ -83,7 +84,7 @@ public class LinkCollectorAPIImpl implements LinkCollectorAPI {
String availabilityString = ""; String availabilityString = "";
int onlineCount = 0; int onlineCount = 0;
for (CrawledLink cl : pkg.getChildren()) { for (CrawledLink cl : pkg.getChildren()) {
if (LinkState.ONLINE.equals(cl.getLinkState())) { if (AvailableLinkState.ONLINE.equals(cl.getLinkState())) {
onlineCount++; onlineCount++;
} }
} }
@ -97,7 +98,7 @@ public class LinkCollectorAPIImpl implements LinkCollectorAPI {
if (queryParams._getQueryParam("availabilityCount", Boolean.class, false)) { if (queryParams._getQueryParam("availabilityCount", Boolean.class, false)) {
int onlineCount = 0; int onlineCount = 0;
for (CrawledLink cl : pkg.getChildren()) { for (CrawledLink cl : pkg.getChildren()) {
if (LinkState.ONLINE.equals(cl.getLinkState())) { if (AvailableLinkState.ONLINE.equals(cl.getLinkState())) {
onlineCount++; onlineCount++;
} }
infomap.put("availabilityCount", onlineCount); infomap.put("availabilityCount", onlineCount);
@ -235,7 +236,7 @@ public class LinkCollectorAPIImpl implements LinkCollectorAPI {
private Boolean addLinks(String links, final String finalPackageName, String extractPassword, final String downloadPassword, final String destinationFolder, final boolean autostart) { private Boolean addLinks(String links, final String finalPackageName, String extractPassword, final String downloadPassword, final String destinationFolder, final boolean autostart) {
LinkCollector lc = LinkCollector.getInstance(); LinkCollector lc = LinkCollector.getInstance();
LinkCollectingJob lcj = new LinkCollectingJob(LinkOrigin.MYJD, links); LinkCollectingJob lcj = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.MYJD, null), links);
HashSet<String> extPws = null; HashSet<String> extPws = null;
if (StringUtils.isNotEmpty(extractPassword)) { if (StringUtils.isNotEmpty(extractPassword)) {
extPws = new HashSet<String>(); extPws = new HashSet<String>();

11
src/org/jdownloader/api/linkcollector/v2/AddLinksQueryStorable.java Normal file
View File

@ -0,0 +1,11 @@
package org.jdownloader.api.linkcollector.v2;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.AddLinksQuery;
public class AddLinksQueryStorable extends AddLinksQuery implements Storable {
public AddLinksQueryStorable(/* Storable */) {
}
}

19
src/org/jdownloader/api/linkcollector/v2/CrawledLinkAPIStorableV2.java Normal file
View File

@ -0,0 +1,19 @@
package org.jdownloader.api.linkcollector.v2;
import jd.controlling.linkcrawler.CrawledLink;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.CrawledLinkStorable;
public class CrawledLinkAPIStorableV2 extends CrawledLinkStorable implements Storable {
public CrawledLinkAPIStorableV2(/* Storable */) {
}
public CrawledLinkAPIStorableV2(CrawledLink link) {
setName(link.getName());
setUuid(link.getUniqueID().getID());
}
}

11
src/org/jdownloader/api/linkcollector/v2/CrawledLinkQueryStorable.java Normal file
View File

@ -0,0 +1,11 @@
package org.jdownloader.api.linkcollector.v2;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.CrawledLinkQuery;
public class CrawledLinkQueryStorable extends CrawledLinkQuery implements Storable {
public CrawledLinkQueryStorable() {
super(/* Storable */);
}
}

19
src/org/jdownloader/api/linkcollector/v2/CrawledPackageAPIStorableV2.java Normal file
View File

@ -0,0 +1,19 @@
package org.jdownloader.api.linkcollector.v2;
import jd.controlling.linkcrawler.CrawledPackage;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.CrawledPackageStorable;
public class CrawledPackageAPIStorableV2 extends CrawledPackageStorable implements Storable {
public CrawledPackageAPIStorableV2(/* STorable */) {
}
public CrawledPackageAPIStorableV2(CrawledPackage pkg) {
setName(pkg.getName());
setUuid(pkg.getUniqueID().getID());
}
}

11
src/org/jdownloader/api/linkcollector/v2/CrawledPackageQueryStorable.java Normal file
View File

@ -0,0 +1,11 @@
package org.jdownloader.api.linkcollector.v2;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.CrawledPackageQuery;
public class CrawledPackageQueryStorable extends CrawledPackageQuery implements Storable {
public CrawledPackageQueryStorable(/* Storable */) {
}
}

491
src/org/jdownloader/api/linkcollector/v2/LinkCollectorAPIImplV2.java Normal file
View File

@ -0,0 +1,491 @@
package org.jdownloader.api.linkcollector.v2;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.CrawledPackageView;
import jd.controlling.linkcrawler.PackageInfo;
import jd.controlling.packagecontroller.AbstractNode;
import jd.plugins.DownloadLink;
import org.appwork.remoteapi.exceptions.BadParameterException;
import org.appwork.utils.StringUtils;
import org.jdownloader.api.RemoteAPIController;
import org.jdownloader.api.downloads.v2.DownloadsAPIV2Impl;
import org.jdownloader.controlling.linkcrawler.LinkVariant;
import org.jdownloader.gui.packagehistorycontroller.DownloadPathHistoryManager;
import org.jdownloader.gui.views.SelectionInfo;
import org.jdownloader.myjdownloader.client.bindings.interfaces.LinkgrabberInterface;
import org.jdownloader.settings.GeneralSettings;
public class LinkCollectorAPIImplV2 implements LinkCollectorAPIV2 {
public LinkCollectorAPIImplV2() {
RemoteAPIController.validateInterfaces(LinkCollectorAPIV2.class, LinkgrabberInterface.class);
}
@SuppressWarnings("unchecked")
public static <T extends AbstractNode> List<T> convertIdsToObjects(final List<T> ret, long[] linkIds, long[] packageIds) {
final HashSet<Long> linklookUp = DownloadsAPIV2Impl.createLookupSet(linkIds);
final HashSet<Long> packageLookup = DownloadsAPIV2Impl.createLookupSet(packageIds);
LinkCollector dlc = LinkCollector.getInstance();
if (linklookUp != null || packageLookup != null) {
boolean readL = dlc.readLock();
try {
main: for (CrawledPackage pkg : dlc.getPackages()) {
if (packageLookup != null && packageLookup.remove(pkg.getUniqueID().getID())) {
ret.add((T) pkg);
if ((packageLookup == null || packageLookup.size() == 0) && (linklookUp == null || linklookUp.size() == 0)) {
break main;
}
}
if (linklookUp != null) {
boolean readL2 = pkg.getModifyLock().readLock();
try {
for (CrawledLink child : pkg.getChildren()) {
if (linklookUp.remove(child.getUniqueID().getID())) {
ret.add((T) child);
if ((packageLookup == null || packageLookup.size() == 0) && (linklookUp == null || linklookUp.size() == 0)) {
break main;
}
}
}
} finally {
pkg.getModifyLock().readUnlock(readL2);
}
}
}
} finally {
dlc.readUnlock(readL);
}
}
return ret;
}
@Override
public ArrayList<CrawledPackageAPIStorableV2> queryPackages(CrawledPackageQueryStorable queryParams) throws BadParameterException {
ArrayList<CrawledPackageAPIStorableV2> result = new ArrayList<CrawledPackageAPIStorableV2>();
LinkCollector lc = LinkCollector.getInstance();
int startWith = queryParams.getStartAt();
int maxResults = queryParams.getMaxResults();
// filter out packages, if specific packageUUIDs given, else return all packages
List<CrawledPackage> packages;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
packages = getPackagesByID(queryParams.getPackageUUIDs());
} else {
packages = lc.getPackagesCopy();
}
if (startWith > lc.getPackages().size() - 1) return result;
if (startWith < 0) startWith = 0;
if (maxResults < 0) maxResults = lc.getPackages().size();
for (int i = startWith; i < startWith + maxResults; i++) {
final CrawledPackage pkg = packages.get(i);
boolean readL = pkg.getModifyLock().readLock();
try {
CrawledPackageAPIStorableV2 cps = new CrawledPackageAPIStorableV2(pkg);
final CrawledPackageView view = new CrawledPackageView();
view.setItems(pkg.getChildren());
if (queryParams.isSaveTo()) {
cps.setSaveTo(pkg.getRawDownloadFolder());
}
if (queryParams.isBytesTotal()) {
cps.setBytesTotal(view.getFileSize());
}
if (queryParams.isChildCount()) {
cps.setChildCount(view.getItems().size());
}
if (queryParams.isHosts()) {
Set<String> hosts = new HashSet<String>();
for (CrawledLink cl : pkg.getChildren()) {
hosts.add(cl.getHost());
}
cps.setHosts(hosts.toArray(new String[] {}));
}
if (queryParams.isComment()) {
cps.setComment(pkg.getComment());
}
if (queryParams.isAvailableOfflineCount() || queryParams.isAvailableOnlineCount() || queryParams.isAvailableTempUnknownCount() || queryParams.isAvailableUnknownCount()) {
int onlineCount = 0;
int offlineCount = 0;
int tempUnknown = 0;
int unknown = 0;
for (CrawledLink cl : pkg.getChildren()) {
switch (cl.getLinkState()) {
case OFFLINE:
offlineCount++;
break;
case ONLINE:
onlineCount++;
break;
case TEMP_UNKNOWN:
tempUnknown++;
break;
case UNKNOWN:
unknown++;
break;
}
if (queryParams.isAvailableOfflineCount()) {
cps.setOfflineCount(offlineCount);
}
if (queryParams.isAvailableOnlineCount()) {
cps.setOnlineCount(onlineCount);
}
if (queryParams.isAvailableTempUnknownCount()) {
cps.setTempUnknownCount(tempUnknown);
}
if (queryParams.isAvailableUnknownCount()) {
cps.setUnknownCount(unknown);
}
}
}
if (queryParams.isEnabled()) {
boolean enabled = false;
for (CrawledLink dl : pkg.getChildren()) {
if (dl.isEnabled()) {
enabled = true;
break;
}
}
cps.setEnabled(enabled);
}
result.add(cps);
if (i == lc.getPackages().size() - 1) {
break;
}
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
return result;
}
@SuppressWarnings("rawtypes")
@Override
public ArrayList<CrawledLinkAPIStorableV2> queryLinks(CrawledLinkQueryStorable queryParams) throws BadParameterException {
ArrayList<CrawledLinkAPIStorableV2> result = new ArrayList<CrawledLinkAPIStorableV2>();
LinkCollector lc = LinkCollector.getInstance();
List<CrawledPackage> matched = null;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
matched = getPackagesByID(queryParams.getPackageUUIDs());
} else {
matched = lc.getPackagesCopy();
}
// collect children of the selected packages and convert to storables for response
List<CrawledLink> links = new ArrayList<CrawledLink>();
for (CrawledPackage pkg : matched) {
boolean readL = pkg.getModifyLock().readLock();
try {
links.addAll(pkg.getChildren());
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
if (links.isEmpty()) return result;
int startWith = queryParams.getStartAt();
int maxResults = queryParams.getMaxResults();
if (startWith > links.size() - 1) return result;
if (startWith < 0) startWith = 0;
if (maxResults < 0) maxResults = links.size();
for (int i = startWith; i < Math.min(startWith + maxResults, links.size()); i++) {
CrawledLink cl = links.get(i);
CrawledLinkAPIStorableV2 cls = new CrawledLinkAPIStorableV2(cl);
if (queryParams.isVariants()) {
cls.setVariants(cl.hasVariantSupport());
}
if (queryParams.isBytesTotal()) {
cls.setBytesTotal(cl.getSize());
}
if (queryParams.isHost()) {
cls.setHost(cl.getHost());
}
if (queryParams.isAvailability()) {
cls.setAvailability(cl.getLinkState());
}
if (queryParams.isUrl()) {
cls.setUrl(cl.getURL());
}
if (queryParams.isEnabled()) {
cls.setEnabled(cl.isEnabled());
}
cls.setPackageUUID(cl.getParentNode().getUniqueID().getID());
result.add(cls);
}
return result;
}
@Override
public int getPackageCount() {
return LinkCollector.getInstance().getPackages().size();
}
@Override
public void addLinks(final AddLinksQueryStorable query) {
LinkCollector lc = LinkCollector.getInstance();
LinkCollectingJob lcj = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.MYJD, null/* add useragent? */), query.getLinks());
HashSet<String> extPws = null;
if (StringUtils.isNotEmpty(query.getExtractPassword())) {
extPws = new HashSet<String>();
extPws.add(query.getExtractPassword());
}
final HashSet<String> finalExtPws = extPws;
lcj.setCrawledLinkModifier(new CrawledLinkModifier() {
private PackageInfo getPackageInfo(CrawledLink link) {
PackageInfo packageInfo = link.getDesiredPackageInfo();
if (packageInfo != null) return packageInfo;
packageInfo = new PackageInfo();
link.setDesiredPackageInfo(packageInfo);
return packageInfo;
}
@Override
public void modifyCrawledLink(CrawledLink link) {
if (finalExtPws != null && finalExtPws.size() > 0) {
link.getArchiveInfo().getExtractionPasswords().addAll(finalExtPws);
}
if (StringUtils.isNotEmpty(query.getPackageName())) {
getPackageInfo(link).setName(query.getPackageName());
getPackageInfo(link).setUniqueId(null);
}
if (StringUtils.isNotEmpty(query.getDestinationFolder())) {
getPackageInfo(link).setDestinationFolder(query.getDestinationFolder());
getPackageInfo(link).setUniqueId(null);
}
DownloadLink dlLink = link.getDownloadLink();
if (dlLink != null) {
if (StringUtils.isNotEmpty(query.getDownloadPassword())) dlLink.setDownloadPassword(query.getDownloadPassword());
}
if (query.isAutostart()) {
link.setAutoConfirmEnabled(true);
link.setAutoStartEnabled(true);
}
}
});
lc.addCrawlerJob(lcj);
}
// @Override
// public boolean uploadLinkContainer(RemoteAPIRequest request) {
// if (request.getRequestType() == REQUESTTYPE.POST) {
// PostRequest post = (PostRequest) request.getHttpRequest();
// }
// return false;
// }
@Override
public long getChildrenChanged(long structureWatermark) {
LinkCollector lc = LinkCollector.getInstance();
if (lc.getChildrenChanges() != structureWatermark) {
return lc.getChildrenChanges();
} else {
return -1l;
}
}
/**
* the SelectionInfo Class is actually used for the GUI downloadtable. it generates a logic selection out of selected links and
* packages.
*
* example: if a package is selected, and non if it's links - all its links will be in the selection info<br>
* example2: if a package is selected AND SOME of it's children. The packge will not be considered as fully selected. only the actual
* selected links.
*
* @param linkIds
* @param packageIds
* @return
* @throws BadParameterException
*/
public static SelectionInfo<CrawledPackage, CrawledLink> getSelectionInfo(long[] linkIds, long[] packageIds) throws BadParameterException {
ArrayList<AbstractNode> list = new ArrayList<AbstractNode>();
if (packageIds != null) {
List<CrawledPackage> packages = getPackagesByID(packageIds);
list.addAll(packages);
}
if (linkIds != null) {
List<CrawledLink> links = getLinksById(linkIds);
list.addAll(links);
}
return new SelectionInfo<CrawledPackage, CrawledLink>(null, list, false);
}
@Override
public void moveToDownloadlist(final long[] linkIds, final long[] packageIds) throws BadParameterException {
LinkCollector.getInstance().moveLinksToDownloadList(getSelectionInfo(linkIds, packageIds));
}
@Override
public void removeLinks(final long[] linkIds, final long[] packageIds) throws BadParameterException {
LinkCollector lc = LinkCollector.getInstance();
lc.writeLock();
try {
lc.removeChildren(getSelectionInfo(linkIds, packageIds).getChildren());
} finally {
lc.writeUnlock();
}
}
@Override
public void renameLink(long linkId, String newName) throws BadParameterException {
CrawledLink lc = getLinkById(linkId);
lc.setName(newName);
}
@Override
public void renamePackage(long packageId, String newName) throws BadParameterException {
CrawledPackage lc = getPackageByID(packageId);
lc.setName(newName);
}
@Override
public void setEnabled(boolean enabled, final long[] linkIds, final long[] packageIds) throws BadParameterException {
try {
LinkCollector.getInstance().writeLock();
List<CrawledLink> sdl = getSelectionInfo(linkIds, packageIds).getChildren();
for (CrawledLink dl : sdl) {
dl.setEnabled(enabled);
}
} finally {
LinkCollector.getInstance().writeUnlock();
}
}
@Override
public void movePackages(long[] packageIds, long afterDestPackageId) throws BadParameterException {
List<CrawledPackage> selectedPackages = getPackagesByID(packageIds);
CrawledPackage afterDestPackage = getPackageByID(afterDestPackageId);
LinkCollector.getInstance().move(selectedPackages, afterDestPackage);
}
private static CrawledPackage getPackageByID(long afterDestPackageId) throws BadParameterException {
CrawledPackage ret = LinkCollector.getInstance().getPackageByID(afterDestPackageId);
if (ret == null) throw new BadParameterException("PackageID Unknown");
return ret;
}
private static List<CrawledPackage> getPackagesByID(long[] packageIds) throws BadParameterException {
List<CrawledPackage> ret = LinkCollector.getInstance().getPackagesByID(packageIds);
if (ret.size() != packageIds.length) { throw new BadParameterException("One or more PackageIDs Unknown"); }
return ret;
}
@Override
@SuppressWarnings("unchecked")
public void moveLinks(long[] linkIds, long afterLinkID, long destPackageID) throws BadParameterException {
LinkCollector dlc = LinkCollector.getInstance();
List<CrawledLink> selectedLinks = getLinksById(linkIds);
CrawledLink afterLink = getLinkById(afterLinkID);
CrawledPackage destpackage = getPackageByID(destPackageID);
dlc.move(selectedLinks, destpackage, afterLink);
}
private static CrawledLink getLinkById(long linkIds) throws BadParameterException {
CrawledLink ret = LinkCollector.getInstance().getLinkByID(linkIds);
if (ret == null) { throw new BadParameterException("LinkID Unknown"); }
return ret;
}
private static List<CrawledLink> getLinksById(long[] linkIds) throws BadParameterException {
List<CrawledLink> ret = LinkCollector.getInstance().getLinksByID(linkIds);
if (ret.size() != linkIds.length) { throw new BadParameterException("One or more LinkIDs Unknown"); }
return ret;
}
@Override
public List<String> getDownloadFolderHistorySelectionBase() {
return DownloadPathHistoryManager.getInstance().listPathes(org.appwork.storage.config.JsonConfig.create(GeneralSettings.class).getDefaultDownloadFolder());
}
@Override
public List<LinkVariantStorableV2> getVariants(long linkid) throws BadParameterException {
ArrayList<LinkVariantStorableV2> ret = new ArrayList<LinkVariantStorableV2>();
CrawledLink cl = getLinkById(linkid);
for (LinkVariant lv : cl.getDownloadLink().getDefaultPlugin().getVariantsByLink(cl.getDownloadLink())) {
ret.add(new LinkVariantStorableV2(lv.getUniqueId(), lv.getName()));
}
return ret;
}
@Override
public void setVariant(long linkid, String variantID) throws BadParameterException {
CrawledLink cl = getLinkById(linkid);
for (LinkVariant lv : cl.getDownloadLink().getDefaultPlugin().getVariantsByLink(cl.getDownloadLink())) {
if (lv.getUniqueId().equals(variantID)) {
cl.getDownloadLink().getDefaultPlugin().setActiveVariantByLink(cl.getDownloadLink(), lv);
return;
}
}
throw new BadParameterException("Unknown variantID");
}
}

42
src/org/jdownloader/api/linkcollector/v2/LinkCollectorAPIV2.java Normal file
View File

@ -0,0 +1,42 @@
package org.jdownloader.api.linkcollector.v2;
import java.util.ArrayList;
import java.util.List;
import org.appwork.remoteapi.RemoteAPIInterface;
import org.appwork.remoteapi.annotations.ApiNamespace;
import org.appwork.remoteapi.exceptions.BadParameterException;
@ApiNamespace("linkgrabberv2")
public interface LinkCollectorAPIV2 extends RemoteAPIInterface {
ArrayList<CrawledPackageAPIStorableV2> queryPackages(CrawledPackageQueryStorable queryParams) throws BadParameterException;
ArrayList<CrawledLinkAPIStorableV2> queryLinks(CrawledLinkQueryStorable queryParams) throws BadParameterException;
void moveToDownloadlist(long[] linkIds, long[] packageIds) throws BadParameterException;
void removeLinks(long[] packageIds, long[] linkIds) throws BadParameterException;
void setEnabled(boolean enabled, long[] linkIds, long[] packageIds) throws BadParameterException;
void renameLink(long linkId, String newName) throws BadParameterException;
void renamePackage(long packageId, String newName) throws BadParameterException;
long getChildrenChanged(long structureWatermark);
List<String> getDownloadFolderHistorySelectionBase();
int getPackageCount();
void movePackages(long[] packageIds, long afterDestPackageId) throws BadParameterException;
void moveLinks(long[] linkIds, long afterLinkID, long destPackageID) throws BadParameterException;
void addLinks(AddLinksQueryStorable query);
List<LinkVariantStorableV2> getVariants(long linkid) throws BadParameterException;
void setVariant(long linkid, String variantID) throws BadParameterException;
}

17
src/org/jdownloader/api/linkcollector/v2/LinkVariantStorableV2.java Normal file
View File

@ -0,0 +1,17 @@
package org.jdownloader.api.linkcollector.v2;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.LinkVariantStorable;
public class LinkVariantStorableV2 extends LinkVariantStorable implements Storable {
public LinkVariantStorableV2(/* storable */) {
}
public LinkVariantStorableV2(String uniqueId, String name) {
super();
setId(uniqueId);
setName(name);
}
}

34
src/org/jdownloader/api/test/LinkgrabberTest.java Normal file
View File

@ -0,0 +1,34 @@
package org.jdownloader.api.test;
import org.appwork.storage.Storage;
import org.jdownloader.api.test.TestClient.Test;
import org.jdownloader.myjdownloader.client.AbstractMyJDClientForDesktopJVM;
import org.jdownloader.myjdownloader.client.bindings.CrawledLinkQuery;
import org.jdownloader.myjdownloader.client.bindings.CrawledLinkStorable;
import org.jdownloader.myjdownloader.client.bindings.LinkVariantStorable;
import org.jdownloader.myjdownloader.client.bindings.interfaces.LinkgrabberInterface;
public class LinkgrabberTest extends Test {
@Override
public void run(Storage config, AbstractMyJDClientForDesktopJVM api) throws Exception {
LinkgrabberInterface link = api.link(LinkgrabberInterface.class, chooseDevice(api));
CrawledLinkStorable[] smallList = link.queryLinks(new CrawledLinkQuery());
// CrawledLinkQuery query = new CrawledLinkQuery();
//
// query.setBytesTotal(true);
// query.setEnabled(true);
//
// query.setHost(true);
//
// query.setUrl(true);
// CrawledLinkStorable[] bigList = link.queryLinks(query);
// CrawledPackageQuery pq = new CrawledPackageQuery();
// pq.setHosts(true);
// link.queryPackages(pq);
CrawledLinkStorable cl = smallList[0];
LinkVariantStorable[] variants = link.getVariants(cl.getUuid());
link.setVariant(cl.getUuid(), variants[0].getId());
System.out.println(1);
}
}

1
src/org/jdownloader/api/test/TestClient.java
View File

@ -170,6 +170,7 @@ public class TestClient {
register(new AccountTest()); register(new AccountTest());
register(new EventsTest()); register(new EventsTest());
register(new DownloadListTest()); register(new DownloadListTest());
register(new LinkgrabberTest());
register(new KeepAliveTest()); register(new KeepAliveTest());
register(new ChangePasswordTest()); register(new ChangePasswordTest());

3
src/org/jdownloader/api/toolbar/JDownloaderToolBarAPIImpl.java
View File

@ -16,6 +16,7 @@ import jd.controlling.linkchecker.LinkCheckerHandler;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.BrokenCrawlerHandler; import jd.controlling.linkcrawler.BrokenCrawlerHandler;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.LinkCrawler; import jd.controlling.linkcrawler.LinkCrawler;
@ -366,7 +367,7 @@ public class JDownloaderToolBarAPIImpl implements JDownloaderToolBarAPI, StateEv
public void addCompleteDom(final String url, final String dom, CrawledLink link) { public void addCompleteDom(final String url, final String dom, CrawledLink link) {
final LinkCollectingJob job = new LinkCollectingJob(LinkOrigin.TOOLBAR, dom); final LinkCollectingJob job = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.TOOLBAR, null), dom);
job.setCustomSourceUrl(url); job.setCustomSourceUrl(url);
AddLinksProgress d = new AddLinksProgress(job) { AddLinksProgress d = new AddLinksProgress(job) {
protected String getSearchInText() { protected String getSearchInText() {

4
src/org/jdownloader/api/toolbar/LinkStatus.java
View File

@ -1,9 +1,9 @@
package org.jdownloader.api.toolbar; package org.jdownloader.api.toolbar;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import org.appwork.storage.Storable; import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
public class LinkStatus implements Storable { public class LinkStatus implements Storable {
@ -34,7 +34,7 @@ public class LinkStatus implements Storable {
return link.getURL(); return link.getURL();
} }
public LinkState getStatus() { public AvailableLinkState getStatus() {
return link.getLinkState(); return link.getLinkState();
} }

17
src/org/jdownloader/controlling/filter/CompiledOnlineStatusFiler.java
View File

@ -1,6 +1,7 @@
package org.jdownloader.controlling.filter; package org.jdownloader.controlling.filter;
import jd.controlling.linkcrawler.CrawledLink.LinkState; import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
import jd.gui.swing.jdgui.views.settings.panels.linkgrabberfilter.editdialog.OnlineStatusFilter; import jd.gui.swing.jdgui.views.settings.panels.linkgrabberfilter.editdialog.OnlineStatusFilter;
public class CompiledOnlineStatusFiler extends OnlineStatusFilter { public class CompiledOnlineStatusFiler extends OnlineStatusFilter {
@ -10,26 +11,26 @@ public class CompiledOnlineStatusFiler extends OnlineStatusFilter {
} }
public boolean matches(LinkState linkState) { public boolean matches(AvailableLinkState linkState) {
switch (getMatchType()) { switch (getMatchType()) {
case IS: case IS:
switch (getOnlineStatus()) { switch (getOnlineStatus()) {
case OFFLINE: case OFFLINE:
return linkState == LinkState.OFFLINE; return linkState == AvailableLinkState.OFFLINE;
case ONLINE: case ONLINE:
return linkState == LinkState.ONLINE; return linkState == AvailableLinkState.ONLINE;
case UNCHECKABLE: case UNCHECKABLE:
return linkState == LinkState.TEMP_UNKNOWN; return linkState == AvailableLinkState.TEMP_UNKNOWN;
} }
return false; return false;
case ISNOT: case ISNOT:
switch (getOnlineStatus()) { switch (getOnlineStatus()) {
case OFFLINE: case OFFLINE:
return linkState != LinkState.OFFLINE; return linkState != AvailableLinkState.OFFLINE;
case ONLINE: case ONLINE:
return linkState != LinkState.ONLINE; return linkState != AvailableLinkState.ONLINE;
case UNCHECKABLE: case UNCHECKABLE:
return linkState != LinkState.TEMP_UNKNOWN; return linkState != AvailableLinkState.TEMP_UNKNOWN;
} }
} }

12
src/org/jdownloader/controlling/filter/RuleWrapper.java
View File

@ -3,11 +3,11 @@ package org.jdownloader.controlling.filter;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.plugins.DownloadLink; import jd.plugins.DownloadLink;
import org.appwork.utils.Files; import org.appwork.utils.Files;
import org.appwork.utils.StringUtils; import org.appwork.utils.StringUtils;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
public class RuleWrapper<T extends FilterRule> { public class RuleWrapper<T extends FilterRule> {
@ -172,7 +172,7 @@ public class RuleWrapper<T extends FilterRule> {
public boolean checkFileType(CrawledLink link) { public boolean checkFileType(CrawledLink link) {
if (getFiletypeFilter() != null) { if (getFiletypeFilter() != null) {
if (link.getLinkState() != LinkState.ONLINE) return false; if (link.getLinkState() != AvailableLinkState.ONLINE) return false;
String ext = Files.getExtension(link.getName()); String ext = Files.getExtension(link.getName());
// if there is no extension, this filter does not match // if there is no extension, this filter does not match
if (ext == null) return false; if (ext == null) return false;
@ -184,7 +184,7 @@ public class RuleWrapper<T extends FilterRule> {
public boolean checkFileSize(CrawledLink link) { public boolean checkFileSize(CrawledLink link) {
if (getFilesizeRule() != null) { if (getFilesizeRule() != null) {
// if (link.getDownloadLink().getView().getBytesTotal() <= 0) return true; // if (link.getDownloadLink().getView().getBytesTotal() <= 0) return true;
if (link.getLinkState() != LinkState.ONLINE) return false; if (link.getLinkState() != AvailableLinkState.ONLINE) return false;
return getFilesizeRule().matches(link.getSize()); return getFilesizeRule().matches(link.getSize());
} }
return true; return true;
@ -210,7 +210,7 @@ public class RuleWrapper<T extends FilterRule> {
public boolean checkFileName(CrawledLink link) { public boolean checkFileName(CrawledLink link) {
if (getFileNameRule() != null) { if (getFileNameRule() != null) {
if (link.getLinkState() != LinkState.ONLINE) return false; if (link.getLinkState() != AvailableLinkState.ONLINE) return false;
return getFileNameRule().matches(link.getName()); return getFileNameRule().matches(link.getName());
} }
@ -249,7 +249,7 @@ public class RuleWrapper<T extends FilterRule> {
} }
public boolean checkOnlineStatus(CrawledLink link) { public boolean checkOnlineStatus(CrawledLink link) {
if (LinkState.UNKNOWN == link.getLinkState()) return false; if (AvailableLinkState.UNKNOWN == link.getLinkState()) return false;
if (getOnlineStatusFilter() != null) { return getOnlineStatusFilter().matches(link.getLinkState()); } if (getOnlineStatusFilter() != null) { return getOnlineStatusFilter().matches(link.getLinkState()); }
return true; return true;
} }
@ -258,7 +258,7 @@ public class RuleWrapper<T extends FilterRule> {
if (getOriginFilter() != null) { if (getOriginFilter() != null) {
if (link == null || link.getOrigin() == null) return false; if (link == null || link.getOrigin() == null) return false;
return getOriginFilter().matches(link.getOrigin()); return getOriginFilter().matches(link.getOrigin().getOrigin());
} }
return true; return true;
} }

1
src/org/jdownloader/controlling/linkcrawler/LinkVariant.java
View File

@ -4,6 +4,7 @@ import javax.swing.Icon;
public interface LinkVariant { public interface LinkVariant {
// //
String getUniqueId();
String getName(); String getName();

2
src/org/jdownloader/gui/notify/linkcrawler/LinkCrawlerBubble.java
View File

@ -65,7 +65,7 @@ public class LinkCrawlerBubble extends AbstractNotifyWindow<LinkCrawlerBubbleCon
if (crawler instanceof JobLinkCrawler) { if (crawler instanceof JobLinkCrawler) {
JobLinkCrawler jlc = (JobLinkCrawler) crawler; JobLinkCrawler jlc = (JobLinkCrawler) crawler;
LinkOrigin src = jlc.getJob().getOrigin(); LinkOrigin src = jlc.getJob().getOrigin().getOrigin();
if (src == null) { if (src == null) {
setHeaderText(_GUI._.LinkCrawlerBubble_update_header()); setHeaderText(_GUI._.LinkCrawlerBubble_update_header());

4
src/org/jdownloader/gui/views/linkgrabber/LinkGrabberTableModel.java
View File

@ -4,7 +4,6 @@ import java.util.List;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.controlling.linkcrawler.CrawledPackage; import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.packagecontroller.AbstractNode; import jd.controlling.packagecontroller.AbstractNode;
@ -26,6 +25,7 @@ import org.jdownloader.gui.views.linkgrabber.columns.DownloadFolderColumn;
import org.jdownloader.gui.views.linkgrabber.columns.PartColumn; import org.jdownloader.gui.views.linkgrabber.columns.PartColumn;
import org.jdownloader.gui.views.linkgrabber.columns.UrlColumn; import org.jdownloader.gui.views.linkgrabber.columns.UrlColumn;
import org.jdownloader.gui.views.linkgrabber.columns.VariantColumn; import org.jdownloader.gui.views.linkgrabber.columns.VariantColumn;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
public class LinkGrabberTableModel extends PackageControllerTableModel<CrawledPackage, CrawledLink> { public class LinkGrabberTableModel extends PackageControllerTableModel<CrawledPackage, CrawledLink> {
@ -59,7 +59,7 @@ public class LinkGrabberTableModel extends PackageControllerTableModel<CrawledPa
boolean autoConfirm = ret.size() > 0 && org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.LINKGRABBER_AUTO_CONFIRM_ENABLED.getValue(); boolean autoConfirm = ret.size() > 0 && org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.LINKGRABBER_AUTO_CONFIRM_ENABLED.getValue();
if (!autoConfirm) { if (!autoConfirm) {
for (CrawledLink l : ret.getAllChildrenNodes()) { for (CrawledLink l : ret.getAllChildrenNodes()) {
if (l.getLinkState() != LinkState.OFFLINE) { if (l.getLinkState() != AvailableLinkState.OFFLINE) {
if (l.isAutoConfirmEnabled()) { if (l.isAutoConfirmEnabled()) {
autoConfirm = true; autoConfirm = true;
break; break;

3
src/org/jdownloader/gui/views/linkgrabber/actions/AddContainerAction.java
View File

@ -7,6 +7,7 @@ import java.io.File;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.nutils.io.JDFileFilter; import jd.nutils.io.JDFileFilter;
import org.appwork.utils.swing.dialog.Dialog; import org.appwork.utils.swing.dialog.Dialog;
@ -61,7 +62,7 @@ public class AddContainerAction extends CustomizableAppAction {
sb.append("file://"); sb.append("file://");
sb.append(r.getAbsolutePath()); sb.append(r.getAbsolutePath());
} }
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.ADD_CONTAINER_ACTION, sb.toString())); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.ADD_CONTAINER_ACTION, null), sb.toString()));
} catch (DialogNoAnswerException e1) { } catch (DialogNoAnswerException e1) {
} }
} }

3
src/org/jdownloader/gui/views/linkgrabber/addlinksdialog/AddLinksDialog.java
View File

@ -33,6 +33,7 @@ import jd.controlling.ClipboardMonitoring;
import jd.controlling.ClipboardMonitoring.ClipboardContent; import jd.controlling.ClipboardMonitoring.ClipboardContent;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier; import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.PackageInfo; import jd.controlling.linkcrawler.PackageInfo;
@ -216,7 +217,7 @@ public class AddLinksDialog extends AbstractDialog<LinkCollectingJob> {
@Override @Override
protected LinkCollectingJob createReturnValue() { protected LinkCollectingJob createReturnValue() {
LinkCollectingJob ret = new LinkCollectingJob(LinkOrigin.ADD_LINKS_DIALOG, input.getText()); LinkCollectingJob ret = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.ADD_LINKS_DIALOG, null), input.getText());
final boolean overwritePackagizerRules = isOverwritePackagizerEnabled(); final boolean overwritePackagizerRules = isOverwritePackagizerEnabled();
final String finalPackageName = packagename.getText().trim(); final String finalPackageName = packagename.getText().trim();
if (StringUtils.isNotEmpty(finalPackageName)) PackageHistoryManager.getInstance().add(finalPackageName); if (StringUtils.isNotEmpty(finalPackageName)) PackageHistoryManager.getInstance().add(finalPackageName);

4
src/org/jdownloader/gui/views/linkgrabber/bottombar/AutoConfirmButton.java
View File

@ -19,7 +19,6 @@ import jd.controlling.linkchecker.LinkCheckerListener;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkCollectorCrawler; import jd.controlling.linkcollector.LinkCollectorCrawler;
import jd.controlling.linkcrawler.CrawledLink; import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLink.LinkState;
import jd.controlling.linkcrawler.CrawledPackage; import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.LinkCrawler; import jd.controlling.linkcrawler.LinkCrawler;
import jd.controlling.linkcrawler.LinkCrawlerEvent; import jd.controlling.linkcrawler.LinkCrawlerEvent;
@ -40,6 +39,7 @@ import org.jdownloader.gui.views.linkgrabber.LinkGrabberTableModel;
import org.jdownloader.gui.views.linkgrabber.contextmenu.ConfirmLinksContextAction; import org.jdownloader.gui.views.linkgrabber.contextmenu.ConfirmLinksContextAction;
import org.jdownloader.gui.views.linkgrabber.contextmenu.ConfirmLinksContextAction.AutoStartOptions; import org.jdownloader.gui.views.linkgrabber.contextmenu.ConfirmLinksContextAction.AutoStartOptions;
import org.jdownloader.images.NewTheme; import org.jdownloader.images.NewTheme;
import org.jdownloader.myjdownloader.client.json.AvailableLinkState;
import org.jdownloader.settings.staticreferences.CFG_LINKGRABBER; import org.jdownloader.settings.staticreferences.CFG_LINKGRABBER;
public class AutoConfirmButton extends ExtButton implements ChangeListener, TableModelListener { public class AutoConfirmButton extends ExtButton implements ChangeListener, TableModelListener {
@ -190,7 +190,7 @@ public class AutoConfirmButton extends ExtButton implements ChangeListener, Tabl
boolean autoStart = org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.LINKGRABBER_AUTO_START_ENABLED.isEnabled(); boolean autoStart = org.jdownloader.settings.staticreferences.CFG_LINKGRABBER.LINKGRABBER_AUTO_START_ENABLED.isEnabled();
boolean autoConfirm = CFG_LINKGRABBER.LINKGRABBER_AUTO_CONFIRM_ENABLED.isEnabled(); boolean autoConfirm = CFG_LINKGRABBER.LINKGRABBER_AUTO_CONFIRM_ENABLED.isEnabled();
for (CrawledLink l : LinkGrabberTableModel.getInstance().getAllChildrenNodes()) { for (CrawledLink l : LinkGrabberTableModel.getInstance().getAllChildrenNodes()) {
if (l.getLinkState() == LinkState.OFFLINE) continue; if (l.getLinkState() == AvailableLinkState.OFFLINE) continue;
if (l.isAutoConfirmEnabled() || autoConfirm) { if (l.isAutoConfirmEnabled() || autoConfirm) {
list.add(l); list.add(l);
if (l.isAutoStartEnabled()) autoStart = true; if (l.isAutoStartEnabled()) autoStart = true;

3
src/org/jdownloader/gui/views/linkgrabber/bottombar/PasteLinksAction.java
View File

@ -7,6 +7,7 @@ import jd.controlling.ClipboardMonitoring.ClipboardContent;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.LinkCrawler; import jd.controlling.linkcrawler.LinkCrawler;
import org.appwork.utils.swing.dialog.Dialog; import org.appwork.utils.swing.dialog.Dialog;
@ -59,7 +60,7 @@ public class PasteLinksAction extends CustomizableAppAction implements ActionCon
@Override @Override
public void run() { public void run() {
ClipboardContent content = ClipboardMonitoring.getINSTANCE().getCurrentContent(); ClipboardContent content = ClipboardMonitoring.getINSTANCE().getCurrentContent();
final LinkCollectingJob crawljob = new LinkCollectingJob(LinkOrigin.PASTE_LINKS_ACTION, content != null ? content.getContent() : null); final LinkCollectingJob crawljob = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.PASTE_LINKS_ACTION, null), content != null ? content.getContent() : null);
if (content != null) crawljob.setCustomSourceUrl(content.getBrowserURL()); if (content != null) crawljob.setCustomSourceUrl(content.getBrowserURL());
crawljob.setDeepAnalyse(isDeepDecryptEnabled()); crawljob.setDeepAnalyse(isDeepDecryptEnabled());

3
src/org/jdownloader/gui/views/linkgrabber/contextmenu/PasteContextLinksAction.java
View File

@ -8,6 +8,7 @@ import jd.controlling.ClipboardMonitoring.ClipboardContent;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import jd.controlling.linkcrawler.LinkCrawler; import jd.controlling.linkcrawler.LinkCrawler;
import org.appwork.utils.swing.dialog.Dialog; import org.appwork.utils.swing.dialog.Dialog;
@ -73,7 +74,7 @@ public class PasteContextLinksAction extends CustomizableTableContextAppAction {
@Override @Override
public void run() { public void run() {
ClipboardContent content = ClipboardMonitoring.getINSTANCE().getCurrentContent(); ClipboardContent content = ClipboardMonitoring.getINSTANCE().getCurrentContent();
final LinkCollectingJob crawljob = new LinkCollectingJob(LinkOrigin.PASTE_LINKS_ACTION, content != null ? content.getContent() : null); final LinkCollectingJob crawljob = new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.PASTE_LINKS_ACTION, null), content != null ? content.getContent() : null);
if (content != null) crawljob.setCustomSourceUrl(content.getBrowserURL()); if (content != null) crawljob.setCustomSourceUrl(content.getBrowserURL());
crawljob.setDeepAnalyse(isDeepDecryptEnabled()); crawljob.setDeepAnalyse(isDeepDecryptEnabled());

3
src/org/jdownloader/startup/commands/AddContainerCommand.java
View File

@ -3,6 +3,7 @@ package org.jdownloader.startup.commands;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
public class AddContainerCommand extends AbstractStartupCommand { public class AddContainerCommand extends AbstractStartupCommand {
@ -13,7 +14,7 @@ public class AddContainerCommand extends AbstractStartupCommand {
@Override @Override
public void run(String command, String... parameters) { public void run(String command, String... parameters) {
for (String s : parameters) { for (String s : parameters) {
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.START_PARAMETER, "file://" + s)); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.START_PARAMETER, null), "file://" + s));
} }
} }

3
src/org/jdownloader/startup/commands/AddLinkCommand.java
View File

@ -3,6 +3,7 @@ package org.jdownloader.startup.commands;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
public class AddLinkCommand extends AbstractStartupCommand { public class AddLinkCommand extends AbstractStartupCommand {
@ -13,7 +14,7 @@ public class AddLinkCommand extends AbstractStartupCommand {
@Override @Override
public void run(String command, String... parameters) { public void run(String command, String... parameters) {
for (String s : parameters) { for (String s : parameters) {
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.START_PARAMETER, s)); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.START_PARAMETER, null), s));
} }
} }

3
src/org/jdownloader/startup/commands/FileCommand.java
View File

@ -5,6 +5,7 @@ import java.util.Arrays;
import jd.controlling.linkcollector.LinkCollectingJob; import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector; import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkOrigin; import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcollector.LinkOriginDetails;
import org.appwork.utils.StringUtils; import org.appwork.utils.StringUtils;
@ -21,7 +22,7 @@ public class FileCommand extends AbstractStartupCommand {
for (String s : parameters) { for (String s : parameters) {
if (StringUtils.isNotEmpty(s)) { if (StringUtils.isNotEmpty(s)) {
LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(LinkOrigin.START_PARAMETER, "file://" + s)); LinkCollector.getInstance().addCrawlerJob(new LinkCollectingJob(new LinkOriginDetails(LinkOrigin.START_PARAMETER, null), "file://" + s));
} }
} }