*Plugins: Fixes/Changes/Maintenance*

- XhamsterShorturls: minor changes
- imgsrc: updated filenames, fixed .gif download, updated crawler to try to find correct file extension right away RE forum 96470
- MotherlessComConfig: updated wording of translation
- MotherLessComCrawler: added BubbleNotification if crawler stops early due to user defined limit RE forum 96456
- YumpuCom: fixed RE forum 91111
- ZeroscansCom: added new main domain zscanscom RE forum 91118
- ImgsrcConfig: added config to select preferred image format for imgsrcru items (not yet used to avoid build block) RE forum 96470
- ZipComic: fixed RE forum 96473

git-svn-id: svn://svn.jdownloader.org/jdownloader/trunk@49829 ebf7c1c2-ba36-0410-9fe8-c592906822b4

Former-commit-id: 4d8ac7ba47a5cb47c99131e47ec4d6f31f1e60d2
This commit is contained in:
psp 2024-09-19 13:48:54 +00:00
parent aaad9f8f31
commit 8044667667
9 changed files with 224 additions and 73 deletions

View File

@ -44,6 +44,7 @@ import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
import jd.plugins.PluginForHost;
import jd.plugins.hoster.ImgSrcRu;
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 2, names = {}, urls = {})
public class ImgSrcRuCrawler extends PluginForDecrypt {
@ -348,15 +349,22 @@ public class ImgSrcRuCrawler extends PluginForDecrypt {
}
final String currentLink = br.getURL();
final Set<String> dups = new HashSet<String>();
final String extDefault = "." + ImgSrcRu.getPreferredImageFormat();
for (final String imageid : imageids) {
if (!dups.add(imageid)) {
logger.info("Skipping duplicated item: " + imageid);
continue;
}
final String guessedFileExtension;
if (br.containsHTML("_" + imageid + "[A-Za-z0-9]+\\.gif'")) {
guessedFileExtension = ".gif";
} else {
guessedFileExtension = extDefault;
}
String url = "/" + username + "/" + imageid + ".html";
final DownloadLink img = createDownloadlink("https://decryptedimgsrc.ru" + url);
img.setReferrerUrl(currentLink);
img.setName(imageid + ".jpg");
img.setName(imageid + guessedFileExtension);
img.setAvailable(true);
if (password != null) {
img.setDownloadPassword(password);

View File

@ -330,6 +330,7 @@ public class MotherLessComCrawler extends PluginForDecrypt {
break;
} else if (maxItemsLimit != -1 && ret.size() >= maxItemsLimit) {
logger.info("Stopping because: Reached max items limit of " + maxItemsLimit);
this.displayBubbleNotification(fpName, "Stopping because: Reached user defined limit of: " + maxItemsLimit + "\r\nYou can adjust this limit in the motherless.com plugin settings.");
break;
} else if (this.isAbort()) {
logger.info("Stopping because: Aborted by user");

View File

@ -71,7 +71,7 @@ public class XhamsterShorturls extends PluginForDecrypt {
@Override
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final String url = param.getCryptedUrl().replaceFirst("http://", "https://");
final String url = param.getCryptedUrl().replaceFirst("^http://", "https://");
br.setFollowRedirects(false);
br.getPage(url);
final String finallink = br.getRedirectLocation();

View File

@ -19,8 +19,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.appwork.storage.TypeRef;
import org.appwork.utils.StringUtils;
import org.jdownloader.scripting.JavaScriptEngineFactory;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
@ -33,6 +33,7 @@ import jd.plugins.FilePackage;
import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
import jd.plugins.hoster.DirectHTTP;
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 3, names = { "yumpu.com" }, urls = { "https?://(?:www\\.)?yumpu\\.com/[a-z]{2}/document/read/(\\d+)/([a-z0-9\\-]+)" })
public class YumpuCom extends PluginForDecrypt {
@ -59,7 +60,7 @@ public class YumpuCom extends PluginForDecrypt {
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final Map<String, Object> entries = JavaScriptEngineFactory.jsonToJavaMap(br.toString());
final Map<String, Object> entries = restoreFromString(br.getRequest().getHtmlCode(), TypeRef.MAP);
final Object errorO = entries.get("error");
if (errorO != null) {
/* E.g. {"error":{"reason":"deleted","message":"Document deleted"}} */
@ -77,7 +78,7 @@ public class YumpuCom extends PluginForDecrypt {
final Map<String, Object> images = (Map<String, Object>) document.get("images");
final String basetitle = (String) images.get("title");
if (StringUtils.isEmpty(base_path) || StringUtils.isEmpty(basetitle) || ressourcelist == null || ressourcelist.size() == 0) {
return null;
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final boolean setComment = !StringUtils.isEmpty(description) && !description.equalsIgnoreCase(fpName);
for (int i = 0; i <= ressourcelist.size() - 1; i++) {
@ -112,38 +113,52 @@ public class YumpuCom extends PluginForDecrypt {
} else {
/* Old way */
br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
br.getPage("https://www." + this.getHost() + "/en/document/json/" + fid);
br.getPage("https://www." + this.getHost() + "/en/document/json2/" + fid);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
Map<String, Object> entries = JavaScriptEngineFactory.jsonToJavaMap(br.toString());
final Map<String, Object> entries = restoreFromString(br.getRequest().getHtmlCode(), TypeRef.MAP);
final Object errorO = entries.get("error");
if (errorO != null) {
/* E.g. {"error":{"reason":"deleted","message":"Document deleted"}} */
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
entries = (Map<String, Object>) entries.get("document");
final List<Object> ressourcelist = (List<Object>) entries.get("pages");
final String description = (String) entries.get("description");
String fpName = (String) entries.get("title");
final Map<String, Object> document = (Map<String, Object>) entries.get("document");
final List<Map<String, Object>> ressourcelist = (List<Map<String, Object>>) document.get("pages");
final String description = (String) document.get("description");
String fpName = (String) document.get("title");
if (StringUtils.isEmpty(fpName)) {
/* Fallback */
fpName = url_name;
}
final String base_path = (String) entries.get("base_path");
entries = (Map<String, Object>) entries.get("images");
final String base_title = (String) entries.get("title");
final Map<String, Object> dimensions = (Map<String, Object>) entries.get("dimensions");
final String best_resolution = (String) dimensions.get("big");
if (StringUtils.isEmpty(base_path) || StringUtils.isEmpty(base_title) || StringUtils.isEmpty(best_resolution) || ressourcelist == null || ressourcelist.size() == 0) {
return null;
final String base_path = (String) document.get("base_path");
final Map<String, Object> images = (Map<String, Object>) document.get("images");
final String title = (String) images.get("title");
final Map<String, Object> dimensions = (Map<String, Object>) images.get("dimensions");
String best_resolution = null;
String best_resolution_label = null;
final String[] qualitiesHighestToLowest = new String[] { "big", "large", "medium", "small", "thumb" };
for (final String quality : qualitiesHighestToLowest) {
best_resolution = (String) dimensions.get(quality);
if (best_resolution != null) {
best_resolution_label = quality;
break;
}
}
if (StringUtils.isEmpty(base_path) || StringUtils.isEmpty(title) || StringUtils.isEmpty(best_resolution) || ressourcelist == null || ressourcelist.size() == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final boolean setComment = !StringUtils.isEmpty(description) && !description.equalsIgnoreCase(fpName);
for (int i = 1; i <= ressourcelist.size(); i++) {
for (int i = 0; i < ressourcelist.size(); i++) {
/* 2019-05-21: 'quality' parameter = percentage of quality */
final String directurl = base_path + i + "/" + best_resolution + "/" + base_title + "?quality=100";
final String filename = i + "_" + base_title;
final DownloadLink dl = createDownloadlink(directurl);
final Map<String, Object> ressource = ressourcelist.get(i);
final Map<String, Object> this_images = (Map<String, Object>) ressource.get("images");
final Map<String, Object> this_qss = (Map<String, Object>) ressource.get("qss");
final String this_image_best_resolution = this_images.get(best_resolution_label).toString();
final String this_image_best_resolution_aws_auth = this_qss.get(best_resolution_label).toString();
final String directurl = base_path + this_image_best_resolution + "?" + this_image_best_resolution_aws_auth;
final String filename = i + "_" + title;
final DownloadLink dl = createDownloadlink(DirectHTTP.createURLForThisPlugin(directurl));
dl.setFinalFileName(filename);
dl.setAvailable(true);
if (setComment) {

View File

@ -40,7 +40,7 @@ import jd.plugins.PluginForDecrypt;
import jd.plugins.components.PluginJSonUtils;
import jd.plugins.hoster.DirectHTTP;
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 3, names = { "zeroscans.com" }, urls = { "https?://(?:www\\.)?zeroscans\\.com/comics/([a-z0-9\\-]+)(/(\\d+))?" })
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 3, names = {}, urls = {})
public class ZeroscansCom extends PluginForDecrypt {
public ZeroscansCom(PluginWrapper wrapper) {
super(wrapper);
@ -53,9 +53,37 @@ public class ZeroscansCom extends PluginForDecrypt {
return br;
}
public static List<String[]> getPluginDomains() {
final List<String[]> ret = new ArrayList<String[]>();
// each entry in List<String[]> will result in one PluginForDecrypt, Plugin.getHost() will return String[0]->main domain
ret.add(new String[] { "zscans.com", "zeroscans.com" });
return ret;
}
public static String[] getAnnotationNames() {
return buildAnnotationNames(getPluginDomains());
}
@Override
public String[] siteSupportedNames() {
return buildSupportedNames(getPluginDomains());
}
public static String[] getAnnotationUrls() {
return buildAnnotationUrls(getPluginDomains());
}
public static String[] buildAnnotationUrls(final List<String[]> pluginDomains) {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : pluginDomains) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/comics/([a-z0-9\\-]+)(/(\\d+))?");
}
return ret.toArray(new String[0]);
}
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final String contenturl = param.getCryptedUrl();
final String contenturl = param.getCryptedUrl().replaceFirst("^http://", "https://");
br.getPage(contenturl);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
@ -92,7 +120,7 @@ public class ZeroscansCom extends PluginForDecrypt {
if (title != null) {
fp.setName(title);
}
fp.setPackageKey("zeroscanscom://chapter/" + chapterID);
fp.setPackageKey("zeroscans://chapter/" + chapterID);
fp.addLinks(ret);
} else {
/* Crawl all chapters of a comic-series */

View File

@ -16,10 +16,9 @@
package jd.plugins.decrypter;
import java.util.ArrayList;
import java.util.List;
import org.appwork.utils.Regex;
import org.appwork.utils.StringUtils;
import org.jdownloader.plugins.components.antiDDoSForDecrypt;
import org.jdownloader.plugins.controller.LazyPlugin;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
@ -28,36 +27,78 @@ import jd.plugins.CryptedLink;
import jd.plugins.DecrypterPlugin;
import jd.plugins.DownloadLink;
import jd.plugins.FilePackage;
import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
import jd.plugins.hoster.DirectHTTP;
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 2, names = { "zipcomic.com" }, urls = { "https?://(www\\.)?zipcomic\\.com/[^/]+/?" })
public class ZipComic extends antiDDoSForDecrypt {
@DecrypterPlugin(revision = "$Revision$", interfaceVersion = 2, names = {}, urls = {})
public class ZipComic extends PluginForDecrypt {
public ZipComic(PluginWrapper wrapper) {
super(wrapper);
}
@Override
public LazyPlugin.FEATURE[] getFeatures() {
return new LazyPlugin.FEATURE[] { LazyPlugin.FEATURE.IMAGE_GALLERY };
}
private static List<String[]> getPluginDomains() {
final List<String[]> ret = new ArrayList<String[]>();
ret.add(new String[] { "zipcomic.com" });
return ret;
}
public static String[] getAnnotationNames() {
return buildAnnotationNames(getPluginDomains());
}
@Override
public String[] siteSupportedNames() {
return buildSupportedNames(getPluginDomains());
}
public static String[] getAnnotationUrls() {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : getPluginDomains()) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/[\\w\\-]+-issue-[\\w\\-]+");
}
return ret.toArray(new String[0]);
}
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress) throws Exception {
ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final String parameter = param.toString();
br.setFollowRedirects(true);
getPage(parameter);
String fpName = br.getRegex("<title>\\s*Read\\s+online,\\s+Download\\s+zip\\s+([^>]+)\\s+comic").getMatch(0);
if (StringUtils.isEmpty(fpName)) {
fpName = br.getRegex("<title>\\s*Read\\s+online([^>]+)\\s+-\\s+Issue").getMatch(0);
br.getPage(parameter);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final String[] links = br.getRegex("<a[^>]+href\\s*=\\s*[\"']*([^\"'>]+)[\"']*[^>]*><i[^>]+class\\s*=\\s*\"fa fa-download\"[^>]*>").getColumn(0);
if (links != null && links.length > 0) {
for (String link : links) {
if (new Regex(link, "^/[^/]").matches()) {
link = "directhttp://" + br.getURL(link).toString();
}
decryptedLinks.add(createDownloadlink(Encoding.htmlDecode(link)));
}
String title = br.getRegex("<h1 class=push-10>Read online ([^<]+)</h1>").getMatch(0);
final String[] links = br.getRegex("class=img-responsive src=\"(https?://[^\"]+)\" alt=\"[^\"]*Issue #").getColumn(0);
if (links == null || links.length == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
if (StringUtils.isNotEmpty(fpName)) {
final FilePackage fp = FilePackage.getInstance();
fp.setName(Encoding.htmlDecode(fpName.trim()));
fp.addLinks(decryptedLinks);
final String urlpath = br._getURL().getPath();
final String urlSlug = urlpath.substring(1);
final String urlTitle = urlSlug.replace("-", " ").trim();
int index = 0;
for (String link : links) {
link = Encoding.htmlOnlyDecode(link);
final DownloadLink image = createDownloadlink(DirectHTTP.createURLForThisPlugin(link));
image.setName(urlTitle + "_" + (index + 1) + ".jpg");
image.setAvailable(true);
ret.add(image);
index++;
}
return decryptedLinks;
final FilePackage fp = FilePackage.getInstance();
if (title != null && !true) {
fp.setName(Encoding.htmlDecode(title).trim());
} else {
/* Fallback */
fp.setName(urlTitle);
}
fp.addLinks(ret);
return ret;
}
}

View File

@ -15,6 +15,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package jd.plugins.hoster;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@ -22,7 +23,6 @@ import org.appwork.utils.StringUtils;
import jd.PluginWrapper;
import jd.http.Browser;
import jd.http.URLConnectionAdapter;
import jd.nutils.encoding.Encoding;
import jd.parser.Regex;
import jd.parser.html.Form;
@ -109,33 +109,61 @@ public class ImgSrcRu extends PluginForHost {
}
private AvailableStatus requestFileInformation(final DownloadLink link, final boolean isDownload) throws Exception {
final String fileID = this.getFID(link);
final String preferredFormat = getPreferredImageFormat();
final String extDefault = "." + preferredFormat;
if (!link.isNameSet()) {
link.setName(fileID + extDefault);
}
prepBrowser(br);
final String r = getReferer(link);
if (r != null) {
br.getHeaders().put("Referer", "https://" + getHost() + "/");
}
getPage(link.getPluginPatternMatcher(), link);
// final String originalFilename = br.getRegex("fetchpriority='high' alt='([^']+)'>").getMatch(0);
// if (originalFilename != null) {
// link.setFinalFileName(originalFilename);
// }
getDllink();
final String filename;
String originalFilename = br.getRegex("fetchpriority='high' alt='([^']+)'>").getMatch(0);
if (originalFilename != null) {
originalFilename = Encoding.htmlDecode(originalFilename).trim();
filename = fileID + "_" + originalFilename;
} else {
filename = fileID;
}
link.setFinalFileName(this.correctOrApplyFileNameExtension(filename, extDefault, null));
this.dllink = getDllink();
if (StringUtils.isEmpty(dllink)) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
} else if (!isDownload) {
basicLinkCheck(br.cloneBrowser(), br.createGetRequest(dllink), link, link.getName(), ".jpeg");
basicLinkCheck(br.cloneBrowser(), br.createGetRequest(dllink), link, filename, extDefault);
}
return AvailableStatus.TRUE;
}
private void getDllink() {
final String[] formats = new String[] { "jpeg", "webp" };
public static final String getPreferredImageFormat() {
return "jpeg";
}
private String getDllink() {
/* Check for .gif first because otherwise we may grab the .gif static preview image which we don't want to grab. */
final String gifLink = br.getRegex("'big-pic-webp'\\)\\.srcset='([^']+\\.gif)'").getMatch(0);
if (gifLink != null) {
return gifLink;
}
final ArrayList<String> formats = new ArrayList<String>();
final String preferredFormat = getPreferredImageFormat();
formats.add("jpeg");
formats.add("webp");
if (preferredFormat != null) {
formats.remove(preferredFormat);
formats.add(0, preferredFormat);
}
for (final String format : formats) {
dllink = br.getRegex("id='big-pic-" + format + "' srcset='([^<>\"']+)'").getMatch(0);
if (dllink != null) {
break;
return dllink;
}
}
return null;
}
@Override
@ -147,23 +175,9 @@ public class ImgSrcRu extends PluginForHost {
br.setFollowRedirects(true);
dl = new jd.plugins.BrowserAdapter().openDownload(br, link, dllink, true, 1);
handleConnectionErrors(br, dl.getConnection());
this.filenameHandling(link, dl.getConnection());
dl.startDownload();
}
private void filenameHandling(final DownloadLink link, final URLConnectionAdapter con) {
String filenameFromConnection = getFileNameFromConnection(con);
if (filenameFromConnection == null) {
return;
}
final String fileid = this.getFID(link);
if (filenameFromConnection.contains(".")) {
/* Remove domain from header-filename. */
filenameFromConnection = filenameFromConnection.substring(filenameFromConnection.lastIndexOf("."));
}
link.setFinalFileName(fileid + filenameFromConnection);
}
public static boolean isPasswordProtected(final Browser br) {
return br.containsHTML("this album requires password\\s*<") || br.containsHTML(">\\s*Album owner\\s*(</a>)?\\s*has protected it from unauthorized access") || br.containsHTML(">\\s*Album owner\\s*(</a>)?\\s*has protected his work from unauthorized access") || br.containsHTML("enter password to continue:");
}

View File

@ -0,0 +1,44 @@
package org.jdownloader.plugins.components.config;
import org.appwork.storage.config.annotations.AboutConfig;
import org.appwork.storage.config.annotations.DefaultEnumValue;
import org.appwork.storage.config.annotations.DescriptionForConfigEntry;
import org.appwork.storage.config.annotations.LabelInterface;
import org.jdownloader.plugins.config.Order;
import org.jdownloader.plugins.config.PluginConfigInterface;
import org.jdownloader.plugins.config.PluginHost;
import org.jdownloader.plugins.config.Type;
@PluginHost(host = "imgsrc.ru", type = Type.HOSTER)
public interface ImgsrcConfig extends PluginConfigInterface {
public static final TRANSLATION TRANSLATION = new TRANSLATION();
public static class TRANSLATION {
public String getPreferredImageFormat_label() {
return "Preferred image format";
}
}
public static enum ImageFormat implements LabelInterface {
JPEG {
@Override
public String getLabel() {
return "JPEG";
}
},
WEBP {
@Override
public String getLabel() {
return "WEBP";
}
};
}
@AboutConfig
@DefaultEnumValue("JPEG")
@Order(10)
@DescriptionForConfigEntry("If the preferred format is not found, another one will be chosen. Animated images are only available as in GIF format.")
ImageFormat getPreferredImageFormat();
void setPreferredImageFormat(final ImageFormat format);
}

View File

@ -12,7 +12,7 @@ import org.jdownloader.plugins.config.Type;
@PluginHost(host = "motherless.com", type = Type.HOSTER)
public interface MotherlessComConfig extends PluginConfigInterface {
final String text_GroupCrawlerLimit = "Define limit for group crawler (0 = disable group crawler, -1 = unlimited)";
final String text_GroupCrawlerLimit = "Limit for group crawler (0 = disable group crawler, -1 = unlimited)";
final String text_UseTitleAsFilenameIfExtensionFits = "Use title as final filename if it ends with the expected file-extension?";
public static final TRANSLATION TRANSLATION = new TRANSLATION();