From b6aefd383698047b61cb81a5cfdc7a011700e3f1 Mon Sep 17 00:00:00 2001 From: psp Date: Fri, 23 Aug 2024 12:56:07 +0000 Subject: [PATCH] *Plugins: Fixes/Changes/Maintenance* - CivitaiComCrawler: implemented CivitaiComConfig RE forum 96302 git-svn-id: svn://svn.jdownloader.org/jdownloader/trunk@49660 ebf7c1c2-ba36-0410-9fe8-c592906822b4 Former-commit-id: 58eada85ca25d6998ba1d27dea8ebb020f19fddc --- src/jd/plugins/decrypter/CivitaiComCrawler.java | 15 ++++++++++++--- src/jd/plugins/hoster/InstaGramCom.java | 5 ++--- .../components/config/CivitaiComConfig.java | 5 ++++- .../components/config/DiskYandexNetConfig.java | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/jd/plugins/decrypter/CivitaiComCrawler.java b/src/jd/plugins/decrypter/CivitaiComCrawler.java index a408408c02..caa5d4113a 100644 --- a/src/jd/plugins/decrypter/CivitaiComCrawler.java +++ b/src/jd/plugins/decrypter/CivitaiComCrawler.java @@ -24,6 +24,8 @@ import java.util.Map; import org.appwork.storage.TypeRef; import org.appwork.utils.DebugMode; import org.appwork.utils.parser.UrlQuery; +import org.jdownloader.plugins.components.config.CivitaiComConfig; +import org.jdownloader.plugins.config.PluginJsonConfig; import jd.PluginWrapper; import jd.controlling.ProgressController; @@ -139,6 +141,7 @@ public class CivitaiComCrawler extends PluginForDecrypt { ret.add(link); } } else if (itemType.equals("user")) { + final CivitaiComConfig cfg = PluginJsonConfig.get(CivitaiComConfig.class); final FilePackage fp = FilePackage.getInstance(); fp.setName(itemID); fp.setPackageKey("civitai://user/" + itemID); @@ -150,7 +153,8 @@ public class CivitaiComCrawler extends PluginForDecrypt { * is wrong --> Wrong API docs or bug in API.
* Only with the nsfw parameter set to "X", all items will be returned. */ - final int maxItemsPerPage = 10; + final int maxItemsPerPage = cfg.getProfileCrawlerMaxPaginationItems(); + final int paginationSleepMillis = cfg.getProfileCrawlerPaginationSleepMillis(); String nextpage = apiBase + "/images?username=" + itemID + "&limit=" + maxItemsPerPage + "&nsfw=X"; int page = 1; final HashSet dupes = new HashSet(); @@ -182,7 +186,7 @@ public class CivitaiComCrawler extends PluginForDecrypt { ret.add(link); distribute(link); } - logger.info("Crawled page " + page + " | Found items so far: " + ret.size() + " | New items on this page: " + numberofNewItems); + logger.info("Crawled page " + page + " | Found items so far: " + ret.size() + " | New items on this page: " + numberofNewItems + " | Max items per page: " + maxItemsPerPage + " | Pagination sleep millis: " + paginationSleepMillis); final Map metadata = (Map) entries.get("metadata"); if (metadata == null) { logger.info("Stopping because: Metadata map doesnt exist"); @@ -200,7 +204,7 @@ public class CivitaiComCrawler extends PluginForDecrypt { } else { /* Continue to next page */ page++; - this.sleep(1000, param); + this.sleep(paginationSleepMillis, param); continue pagination; } } @@ -260,4 +264,9 @@ public class CivitaiComCrawler extends PluginForDecrypt { public boolean hasCaptcha(CryptedLink link, Account acc) { return false; } + + @Override + public Class getConfigInterface() { + return CivitaiComConfig.class; + } } diff --git a/src/jd/plugins/hoster/InstaGramCom.java b/src/jd/plugins/hoster/InstaGramCom.java index e4b7c95253..2f2b8bbf4c 100644 --- a/src/jd/plugins/hoster/InstaGramCom.java +++ b/src/jd/plugins/hoster/InstaGramCom.java @@ -141,7 +141,6 @@ public class InstaGramCom extends PluginForHost { private final int MAXCHUNKS_pictures = 1; /* 2020-01-21: Multi chunks are possible but it's better not to do this to avoid getting blocked! */ private final int MAXCHUNKS_videos = 1; - private final int MAXDOWNLOADS = -1; private static final String MAINPAGE = "https://www.instagram.com"; /* DownloadLink/Packagizer properties */ public static final String PROPERTY_has_tried_to_crawl_original_url = "has_tried_to_crawl_original_url"; @@ -669,7 +668,7 @@ public class InstaGramCom extends PluginForHost { @Override public int getMaxSimultanFreeDownloadNum() { - return MAXDOWNLOADS; + return Integer.MAX_VALUE; } public void login(final Account account, final boolean force) throws Exception { @@ -1134,7 +1133,7 @@ public class InstaGramCom extends PluginForHost { @Override public int getMaxSimultanPremiumDownloadNum() { - return MAXDOWNLOADS; + return this.getMaxSimultanFreeDownloadNum(); } @Override diff --git a/src/org/jdownloader/plugins/components/config/CivitaiComConfig.java b/src/org/jdownloader/plugins/components/config/CivitaiComConfig.java index 3ab7a861f0..31df582887 100644 --- a/src/org/jdownloader/plugins/components/config/CivitaiComConfig.java +++ b/src/org/jdownloader/plugins/components/config/CivitaiComConfig.java @@ -2,6 +2,7 @@ package org.jdownloader.plugins.components.config; import org.appwork.storage.config.annotations.AboutConfig; import org.appwork.storage.config.annotations.DefaultIntValue; +import org.appwork.storage.config.annotations.DefaultOnNull; import org.appwork.storage.config.annotations.DescriptionForConfigEntry; import org.appwork.storage.config.annotations.SpinnerValidator; import org.jdownloader.plugins.config.Order; @@ -9,7 +10,7 @@ import org.jdownloader.plugins.config.PluginConfigInterface; import org.jdownloader.plugins.config.PluginHost; import org.jdownloader.plugins.config.Type; -@PluginHost(host = "civitai.com", type = Type.HOSTER) +@PluginHost(host = "civitai.com", type = Type.CRAWLER) public interface CivitaiComConfig extends PluginConfigInterface { public static final TRANSLATION TRANSLATION = new TRANSLATION(); @@ -27,6 +28,7 @@ public interface CivitaiComConfig extends PluginConfigInterface { @SpinnerValidator(min = 10, max = 200, step = 1) @DefaultIntValue(50) @DescriptionForConfigEntry("Internal value to limit max number of items per page. Lower value = More requests needed to crawl a profile. See: https://github.com/civitai/civitai/wiki/REST-API-Reference#get-apiv1images") + @DefaultOnNull() @Order(10) int getProfileCrawlerMaxPaginationItems(); @@ -36,6 +38,7 @@ public interface CivitaiComConfig extends PluginConfigInterface { @SpinnerValidator(min = 0, max = 20000, step = 1000) @DefaultIntValue(1000) @DescriptionForConfigEntry("Wait time between pagination requests.") + @DefaultOnNull() @Order(20) int getProfileCrawlerPaginationSleepMillis(); diff --git a/src/org/jdownloader/plugins/components/config/DiskYandexNetConfig.java b/src/org/jdownloader/plugins/components/config/DiskYandexNetConfig.java index 8832d4b5b6..7652d810ca 100644 --- a/src/org/jdownloader/plugins/components/config/DiskYandexNetConfig.java +++ b/src/org/jdownloader/plugins/components/config/DiskYandexNetConfig.java @@ -41,9 +41,9 @@ public interface DiskYandexNetConfig extends PluginConfigInterface { @AboutConfig @DefaultEnumValue("MOVE_INTO_ACCOUNT") - @Order(100) @DescriptionForConfigEntry("What to do with quota limited files in account download mode?") @DefaultOnNull + @Order(100) ActionForQuotaLimitedFiles getActionForQuotaLimitedFiles(); void setActionForQuotaLimitedFiles(final ActionForQuotaLimitedFiles action); @@ -71,9 +71,9 @@ public interface DiskYandexNetConfig extends PluginConfigInterface { @AboutConfig @DefaultEnumValue("DELETE_FROM_ACCOUNT") - @Order(200) @DescriptionForConfigEntry("What to do with files that JDownloader has moved/imported into your Yandex account?") @DefaultOnNull + @Order(200) ActionForMovedFiles getActionForMovedFiles(); void setActionForMovedFiles(final ActionForMovedFiles action);