*Plugins: Fixes/Changes/Maintenance*

- ZipComic: minor changes
- YumpuCom: removed duplicated code
- AhMeComGallery: updated offline detection RE forum 96478 refs #48428
- SaintTo: added support for another linktype RE forum 93269

*Plugins: RIP*
- [XFS] YoudboxCom RE forum 87969

git-svn-id: svn://svn.jdownloader.org/jdownloader/trunk@49835 ebf7c1c2-ba36-0410-9fe8-c592906822b4

Former-commit-id: 2e31bbc373ab4f81c807155beee6471f6cf659a0
This commit is contained in:
psp 2024-09-20 10:33:08 +00:00
parent c1f54f6e2d
commit 75096cb81d
5 changed files with 167 additions and 323 deletions

View File

@ -21,6 +21,7 @@ import org.jdownloader.plugins.controller.LazyPlugin;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
import jd.http.Browser;
import jd.nutils.encoding.Encoding;
import jd.parser.Regex;
import jd.plugins.CryptedLink;
@ -31,34 +32,44 @@ import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 2, names = { "ah-me.com" }, urls = { "https?://(?:www\\.)?ah\\-me\\.com/pics/gallery/\\d+/\\d+/" })
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 2, names = { "ah-me.com" }, urls = { "https?://(?:www\\.)?ah\\-me\\.com/pics/gallery/(\\d+)/(\\d+)/" })
public class AhMeComGallery extends PluginForDecrypt {
public AhMeComGallery(PluginWrapper wrapper) {
super(wrapper);
}
@Override
public Browser createNewBrowserInstance() {
final Browser br = super.createNewBrowserInstance();
br.setFollowRedirects(true);
return br;
}
@Override
public LazyPlugin.FEATURE[] getFeatures() {
return new LazyPlugin.FEATURE[] { LazyPlugin.FEATURE.XXX, LazyPlugin.FEATURE.IMAGE_GALLERY };
}
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress) throws Exception {
ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
br.setFollowRedirects(true);
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
br.getPage(param.getCryptedUrl());
if (br.containsHTML("class=\"gal_thumbs spec_right\">\\s*</div>") || br.getHttpConnection().getResponseCode() == 404) {
final String galleryID = new Regex(param.getCryptedUrl(), this.getSupportedLinks()).getMatch(0);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
} else if (br.containsHTML("class=\"gal_thumbs spec_right\">\\s*</div>")) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
} else if (!br.getURL().contains(galleryID)) {
/* E.g. redirect to main page */
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final String galleryID = new Regex(param.getCryptedUrl(), "(\\d+)/\\d+/").getMatch(0);
String fpName = br.getRegex("<h2>([^<>\"]*?)</h2>").getMatch(0);
if (fpName == null) {
fpName = this.getHost() + " gallery " + galleryID;
}
final String[] links = br.getRegex("class=\"thumb\"[^>]*src=\"(https?://[^/]+/work/[^/]+/[^\"]+\\.jpg)\"").getColumn(0);
if (links == null || links.length == 0) {
logger.warning("Decrypter broken for link: " + param.getCryptedUrl());
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
for (final String singleLink : links) {
final String partToRemove = new Regex(singleLink, "/work/([^/]+)/").getMatch(0);
if (partToRemove == null) {
@ -66,11 +77,11 @@ public class AhMeComGallery extends PluginForDecrypt {
}
final DownloadLink dl = createDownloadlink(singleLink.replaceFirst(org.appwork.utils.Regex.escape(partToRemove), "orig"));
dl.setAvailable(true);
decryptedLinks.add(dl);
ret.add(dl);
}
final FilePackage fp = FilePackage.getInstance();
fp.setName(Encoding.htmlDecode(fpName).trim());
fp.addLinks(decryptedLinks);
return decryptedLinks;
fp.addLinks(ret);
return ret;
}
}

View File

@ -24,6 +24,7 @@ import org.appwork.utils.StringUtils;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
import jd.http.Browser;
import jd.nutils.encoding.Encoding;
import jd.parser.Regex;
import jd.plugins.CryptedLink;
@ -41,76 +42,21 @@ public class YumpuCom extends PluginForDecrypt {
super(wrapper);
}
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
@Override
public Browser createNewBrowserInstance() {
final Browser br = super.createNewBrowserInstance();
br.setFollowRedirects(true);
return br;
}
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
br.getPage(param.getCryptedUrl());
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final String jsonURL = br.getRegex("var jsonUrl = \"([^\"]+)\"").getMatch(0);
final String fid = new Regex(param.getCryptedUrl(), this.getSupportedLinks()).getMatch(0);
final String url_name = new Regex(param.getCryptedUrl(), this.getSupportedLinks()).getMatch(1);
final String[] possibleQualityIdentifiersSorted = new String[] { "big", "large", "medium", "small" };
if (jsonURL != null) {
/* New way */
br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
// br.getPage("https://www." + this.getHost() + "/en/document/json2/" + fid);
br.getPage(jsonURL);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final Map<String, Object> entries = restoreFromString(br.getRequest().getHtmlCode(), TypeRef.MAP);
final Object errorO = entries.get("error");
if (errorO != null) {
/* E.g. {"error":{"reason":"deleted","message":"Document deleted"}} */
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final Map<String, Object> document = (Map<String, Object>) entries.get("document");
final List<Map<String, Object>> ressourcelist = (List<Map<String, Object>>) document.get("pages");
final String description = (String) document.get("description");
String fpName = (String) document.get("title");
if (StringUtils.isEmpty(fpName)) {
/* Fallback */
fpName = url_name;
}
final String base_path = (String) document.get("base_path");
final Map<String, Object> images = (Map<String, Object>) document.get("images");
final String basetitle = (String) images.get("title");
if (StringUtils.isEmpty(base_path) || StringUtils.isEmpty(basetitle) || ressourcelist == null || ressourcelist.size() == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final boolean setComment = !StringUtils.isEmpty(description) && !description.equalsIgnoreCase(fpName);
for (int i = 0; i <= ressourcelist.size() - 1; i++) {
/* 2019-05-21: 'quality' = percentage of quality */
final Map<String, Object> page = ressourcelist.get(i);
final Map<String, Object> imagesInfo1 = (Map<String, Object>) page.get("images");
final Map<String, Object> imagesInfo2 = (Map<String, Object>) page.get("qss");
String urlPath = null, query = null;
for (final String possibleQualityIdentifierSorted : possibleQualityIdentifiersSorted) {
if (imagesInfo1.containsKey(possibleQualityIdentifierSorted) && imagesInfo2.containsKey(possibleQualityIdentifierSorted)) {
// bestResolution = (String) dimensions.get(possibleQualityIdentifierSorted);
urlPath = imagesInfo1.get(possibleQualityIdentifierSorted).toString();
query = imagesInfo2.get(possibleQualityIdentifierSorted).toString();
break;
}
}
final String directurl = base_path + urlPath + "?" + query;
final String filename = i + "_" + basetitle;
final DownloadLink dl = createDownloadlink(directurl + "");
dl.setFinalFileName(filename);
dl.setAvailable(true);
if (setComment) {
dl.setComment(description);
}
ret.add(dl);
}
if (fpName != null) {
final FilePackage fp = FilePackage.getInstance();
fp.setName(Encoding.htmlDecode(fpName).trim());
fp.addLinks(ret);
}
} else {
/* Old way */
br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
br.getPage("https://www." + this.getHost() + "/en/document/json2/" + fid);
@ -137,8 +83,7 @@ public class YumpuCom extends PluginForDecrypt {
final Map<String, Object> dimensions = (Map<String, Object>) images.get("dimensions");
String best_resolution = null;
String best_resolution_label = null;
final String[] qualitiesHighestToLowest = new String[] { "big", "large", "medium", "small", "thumb" };
for (final String quality : qualitiesHighestToLowest) {
for (final String quality : possibleQualityIdentifiersSorted) {
best_resolution = (String) dimensions.get(quality);
if (best_resolution != null) {
best_resolution_label = quality;
@ -148,6 +93,7 @@ public class YumpuCom extends PluginForDecrypt {
if (StringUtils.isEmpty(base_path) || StringUtils.isEmpty(title) || StringUtils.isEmpty(best_resolution) || ressourcelist == null || ressourcelist.size() == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final boolean setComment = !StringUtils.isEmpty(description) && !description.equalsIgnoreCase(fpName);
for (int i = 0; i < ressourcelist.size(); i++) {
/* 2019-05-21: 'quality' parameter = percentage of quality */
@ -171,7 +117,6 @@ public class YumpuCom extends PluginForDecrypt {
fp.setName(Encoding.htmlDecode(fpName.trim()));
fp.addLinks(ret);
}
}
return ret;
}
}

View File

@ -92,7 +92,7 @@ public class ZipComic extends PluginForDecrypt {
index++;
}
final FilePackage fp = FilePackage.getInstance();
if (title != null && !true) {
if (title != null) {
fp.setName(Encoding.htmlDecode(title).trim());
} else {
/* Fallback */

View File

@ -3,6 +3,7 @@ package jd.plugins.hoster;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.appwork.utils.Regex;
import org.appwork.utils.StringUtils;
@ -10,6 +11,7 @@ import org.jdownloader.plugins.controller.LazyPlugin;
import jd.PluginWrapper;
import jd.http.Browser;
import jd.http.URLConnectionAdapter;
import jd.nutils.encoding.Encoding;
import jd.plugins.Account;
import jd.plugins.DownloadLink;
@ -64,10 +66,13 @@ public class SaintTo extends PluginForHost {
return buildSupportedNames(getPluginDomains());
}
private static final Pattern TYPE_EMBED = Pattern.compile("/embed/([A-Za-z0-9\\-_]+)", Pattern.CASE_INSENSITIVE);
private static final Pattern TYPE_API_DIRECT = Pattern.compile("/api/download\\.php\\?file=([a-zA-Z0-9_/\\+\\=\\-%]+)/?", Pattern.CASE_INSENSITIVE);
public static String[] getAnnotationUrls() {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : getPluginDomains()) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/embed/([A-Za-z0-9\\-_]+)");
ret.add("https?://(?:\\w+\\.)?" + buildHostsPatternPart(domains) + "(" + TYPE_EMBED.pattern() + "|" + TYPE_API_DIRECT.pattern() + ")");
}
return ret.toArray(new String[0]);
}
@ -92,11 +97,20 @@ public class SaintTo extends PluginForHost {
}
private String getFID(final DownloadLink link) {
return new Regex(link.getPluginPatternMatcher(), this.getSupportedLinks()).getMatch(0);
String fid = new Regex(link.getPluginPatternMatcher(), TYPE_EMBED).getMatch(0);
if (fid == null) {
fid = new Regex(link.getPluginPatternMatcher(), TYPE_API_DIRECT).getMatch(0);
}
return fid;
}
@Override
public AvailableStatus requestFileInformation(final DownloadLink link) throws IOException, PluginException {
return requestFileInformation(link, false);
}
private AvailableStatus requestFileInformation(final DownloadLink link, final boolean isDownload) throws IOException, PluginException {
if (new Regex(link.getPluginPatternMatcher(), TYPE_EMBED).patternFind()) {
if (!link.isNameSet()) {
/* Fallback */
link.setName(this.getFID(link) + ".mp4");
@ -113,6 +127,16 @@ public class SaintTo extends PluginForHost {
filename = Encoding.htmlDecode(filename).trim();
link.setName(filename);
}
} else {
/* TYPE_API_DIRECT */
final Regex urlinfo = new Regex(link.getPluginPatternMatcher(), TYPE_API_DIRECT);
final String filenameBase64 = urlinfo.getMatch(0);
final String filename = Encoding.htmlDecode(filenameBase64);
link.setName(filename);
if (!isDownload) {
this.basicLinkCheck(br, br.createHeadRequest(link.getPluginPatternMatcher()), link, filename, ".mp4");
}
}
return AvailableStatus.TRUE;
}
@ -122,49 +146,60 @@ public class SaintTo extends PluginForHost {
}
private void handleDownload(final DownloadLink link, final String directlinkproperty) throws Exception, PluginException {
if (!attemptStoredDownloadurlDownload(link, directlinkproperty)) {
requestFileInformation(link);
final String dllink = br.getRegex("<source src\\s*=\\s*\"(https?://[^\"]+)\" type=\"video/mp4\">").getMatch(0);
final String storedDirecturl = directlinkproperty;
final String dllink;
final boolean storeDirecturl;
if (new Regex(link.getPluginPatternMatcher(), TYPE_API_DIRECT).patternFind()) {
dllink = link.getPluginPatternMatcher();
storeDirecturl = false;
} else if (storedDirecturl != null) {
logger.info("Trying to re-use stored directurl: " + storedDirecturl);
dllink = storedDirecturl;
storeDirecturl = false;
} else {
/* TYPE_EMBED */
requestFileInformation(link, true);
dllink = br.getRegex("<source src\\s*=\\s*\"(https?://[^\"]+)\" type=\"video/mp4\">").getMatch(0);
if (StringUtils.isEmpty(dllink)) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT, "Failed to find final downloadurl");
}
storeDirecturl = true;
}
/* Important otherwise we can't re-use direct urls (hotlinking-block)! */
br.getHeaders().put("Referer", link.getPluginPatternMatcher());
try {
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dllink, this.isResumeable(link, null), this.getMaxChunks(link, null));
handleConnectionErrors(br, dl.getConnection());
} catch (final Exception e) {
if (storedDirecturl != null) {
link.removeProperty(directlinkproperty);
throw new PluginException(LinkStatus.ERROR_RETRY, "Stored directurl expired", e);
} else {
throw e;
}
}
if (storeDirecturl) {
link.setProperty(directlinkproperty, dl.getConnection().getURL().toExternalForm());
}
dl.startDownload();
}
@Override
public boolean hasCaptcha(DownloadLink link, jd.plugins.Account acc) {
return false;
protected void handleConnectionErrors(final Browser br, final URLConnectionAdapter con) throws PluginException, IOException {
if (!this.looksLikeDownloadableContent(con)) {
br.followConnection(true);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
throwConnectionExceptions(br, con);
throwFinalConnectionException(br, con);
}
}
private boolean attemptStoredDownloadurlDownload(final DownloadLink link, final String directlinkproperty) throws Exception {
final String url = link.getStringProperty(directlinkproperty);
if (StringUtils.isEmpty(url)) {
@Override
public boolean hasCaptcha(DownloadLink link, jd.plugins.Account acc) {
return false;
}
try {
final Browser brc = br.cloneBrowser();
/* Important otherwise we can't re-use directurls (hotlinking-block)! */
brc.getHeaders().put("Referer", link.getPluginPatternMatcher());
dl = new jd.plugins.BrowserAdapter().openDownload(brc, link, url, this.isResumeable(link, null), this.getMaxChunks(link, null));
if (this.looksLikeDownloadableContent(dl.getConnection())) {
return true;
} else {
brc.followConnection(true);
throw new IOException();
}
} catch (final Throwable e) {
logger.log(e);
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
return false;
}
}
@Override
public int getMaxSimultanFreeDownloadNum() {

View File

@ -1,147 +0,0 @@
//jDownloader - Downloadmanager
//Copyright (C) 2013 JD-Team support@jdownloader.org
//
//This program is free software: you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation, either version 3 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program. If not, see <http://www.gnu.org/licenses/>.
package jd.plugins.hoster;
import java.util.ArrayList;
import java.util.List;
import org.jdownloader.plugins.components.XFileSharingProBasic;
import jd.PluginWrapper;
import jd.http.Browser;
import jd.parser.Regex;
import jd.plugins.Account;
import jd.plugins.Account.AccountType;
import jd.plugins.DownloadLink;
import jd.plugins.HostPlugin;
@HostPlugin(revision = "$Revision$", interfaceVersion = 3, names = {}, urls = {})
public class YoudboxCom extends XFileSharingProBasic {
public YoudboxCom(final PluginWrapper wrapper) {
super(wrapper);
this.enablePremium(super.getPurchasePremiumURL());
}
/**
* DEV NOTES XfileSharingProBasic Version SEE SUPER-CLASS<br />
* mods: See overridden functions<br />
* limit-info:<br />
* captchatype-info: 2019-12-09: reCaptchaV2<br />
* other:<br />
*/
public static List<String[]> getPluginDomains() {
final List<String[]> ret = new ArrayList<String[]>();
// each entry in List<String[]> will result in one PluginForHost, Plugin.getHost() will return String[0]->main domain
ret.add(new String[] { "youdbox.site", "youdboox.com", "youdbox.net", "youdbox.com" });
return ret;
}
@Override
public String rewriteHost(final String host) {
/* 2021-09-24: Main domain changed from youdbox.com to youdbox.net. */
return this.rewriteHost(getPluginDomains(), host);
}
public static String[] getAnnotationNames() {
return buildAnnotationNames(getPluginDomains());
}
@Override
public String[] siteSupportedNames() {
return buildSupportedNames(getPluginDomains());
}
private final String PATTERN_SPECIAL = "(?i)https?://[^/]+/mp3embed-([a-z0-9]{12})";
public static String[] getAnnotationUrls() {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : getPluginDomains()) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "(?::\\d+)?" + "(" + XFileSharingProBasic.getDefaultAnnotationPatternPart() + "|/mp3embed-[a-z0-9]{12})");
}
return ret.toArray(new String[0]);
}
@Override
public boolean isResumeable(final DownloadLink link, final Account account) {
if (account != null && account.getType() == AccountType.FREE) {
/* Free Account */
return false;
} else if (account != null && account.getType() == AccountType.PREMIUM) {
/* Premium account */
return false;
} else {
/* Free(anonymous) and unknown account type */
return false;
}
}
@Override
public int getMaxChunks(final Account account) {
if (account != null && account.getType() == AccountType.FREE) {
/* Free Account */
return 1;
} else if (account != null && account.getType() == AccountType.PREMIUM) {
/* Premium account */
return 1;
} else {
/* Free(anonymous) and unknown account type */
return 1;
}
}
@Override
public int getMaxSimultaneousFreeAnonymousDownloads() {
return 1;
}
@Override
public int getMaxSimultaneousFreeAccountDownloads() {
return 1;
}
@Override
public int getMaxSimultanPremiumDownloadNum() {
return 1;
}
@Override
protected URL_TYPE getURLType(final String url) {
if (url != null && url.matches(PATTERN_SPECIAL)) {
return URL_TYPE.NORMAL;
} else {
return super.getURLType(url);
}
}
@Override
protected String getFUID(final String url, final URL_TYPE type) {
final String idFromSpecialPattern = new Regex(url, PATTERN_SPECIAL).getMatch(0);
if (idFromSpecialPattern != null) {
return idFromSpecialPattern;
} else {
return super.getFUID(url, type);
}
}
@Override
protected boolean isVideohosterEmbedHTML(final Browser br) {
/*
* Special/small workaround: Website looks like videohost but they only host audio files and typically their filenames already end
* with ".mp3".
*/
return false;
}
}