JDownloader2/jdownloader
1
0
Fork 0
mirror of https://github.com/mirror/jdownloader.git synced 2024-11-23 03:59:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

*Plugins: Fixes/Changes/Maintenance*

Browse Source 
- CivitaiComCrawler: implemented CivitaiComConfig RE forum 96302

git-svn-id: svn://svn.jdownloader.org/jdownloader/trunk@49660 ebf7c1c2-ba36-0410-9fe8-c592906822b4

Former-commit-id: 58eada85ca25d6998ba1d27dea8ebb020f19fddc
This commit is contained in:
psp 2024-08-23 12:56:07 +00:00
parent e7ba9a255d
commit b6aefd3836
4 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/jd/plugins/decrypter/CivitaiComCrawler.java
View File

@ -24,6 +24,8 @@ import java.util.Map;
import org.appwork.storage.TypeRef;
import org.appwork.utils.DebugMode;
import org.appwork.utils.parser.UrlQuery;
import org.jdownloader.plugins.components.config.CivitaiComConfig;
import org.jdownloader.plugins.config.PluginJsonConfig;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
@ -139,6 +141,7 @@ public class CivitaiComCrawler extends PluginForDecrypt {
ret.add(link);
}
} else if (itemType.equals("user")) {
final CivitaiComConfig cfg = PluginJsonConfig.get(CivitaiComConfig.class);
final FilePackage fp = FilePackage.getInstance();
fp.setName(itemID);
fp.setPackageKey("civitai://user/" + itemID);
@ -150,7 +153,8 @@ public class CivitaiComCrawler extends PluginForDecrypt {
* is wrong --> Wrong API docs or bug in API. </br>
* Only with the nsfw parameter set to "X", all items will be returned.
*/
final int maxItemsPerPage = 10;
final int maxItemsPerPage = cfg.getProfileCrawlerMaxPaginationItems();
final int paginationSleepMillis = cfg.getProfileCrawlerPaginationSleepMillis();
String nextpage = apiBase + "/images?username=" + itemID + "&limit=" + maxItemsPerPage + "&nsfw=X";
int page = 1;
final HashSet<String> dupes = new HashSet<String>();
@ -182,7 +186,7 @@ public class CivitaiComCrawler extends PluginForDecrypt {
ret.add(link);
distribute(link);
}
logger.info("Crawled page " + page + " | Found items so far: " + ret.size() + " | New items on this page: " + numberofNewItems);
logger.info("Crawled page " + page + " | Found items so far: " + ret.size() + " | New items on this page: " + numberofNewItems + " | Max items per page: " + maxItemsPerPage + " | Pagination sleep millis: " + paginationSleepMillis);
final Map<String, Object> metadata = (Map<String, Object>) entries.get("metadata");
if (metadata == null) {
logger.info("Stopping because: Metadata map doesnt exist");
@ -200,7 +204,7 @@ public class CivitaiComCrawler extends PluginForDecrypt {
} else {
/* Continue to next page */
page++;
this.sleep(1000, param);
this.sleep(paginationSleepMillis, param);
continue pagination;
}
}
@ -260,4 +264,9 @@ public class CivitaiComCrawler extends PluginForDecrypt {
public boolean hasCaptcha(CryptedLink link, Account acc) {
return false;
}
@Override
public Class<? extends CivitaiComConfig> getConfigInterface() {
return CivitaiComConfig.class;
}
}

5
src/jd/plugins/hoster/InstaGramCom.java
View File

@ -141,7 +141,6 @@ public class InstaGramCom extends PluginForHost {
private final int MAXCHUNKS_pictures = 1;
/* 2020-01-21: Multi chunks are possible but it's better not to do this to avoid getting blocked! */
private final int MAXCHUNKS_videos = 1;
private final int MAXDOWNLOADS = -1;
private static final String MAINPAGE = "https://www.instagram.com";
/* DownloadLink/Packagizer properties */
public static final String PROPERTY_has_tried_to_crawl_original_url = "has_tried_to_crawl_original_url";
@ -669,7 +668,7 @@ public class InstaGramCom extends PluginForHost {
@Override
public int getMaxSimultanFreeDownloadNum() {
return MAXDOWNLOADS;
return Integer.MAX_VALUE;
}
public void login(final Account account, final boolean force) throws Exception {
@ -1134,7 +1133,7 @@ public class InstaGramCom extends PluginForHost {
@Override
public int getMaxSimultanPremiumDownloadNum() {
return MAXDOWNLOADS;
return this.getMaxSimultanFreeDownloadNum();
}
@Override

5
src/org/jdownloader/plugins/components/config/CivitaiComConfig.java
View File

@ -2,6 +2,7 @@ package org.jdownloader.plugins.components.config;
import org.appwork.storage.config.annotations.AboutConfig;
import org.appwork.storage.config.annotations.DefaultIntValue;
import org.appwork.storage.config.annotations.DefaultOnNull;
import org.appwork.storage.config.annotations.DescriptionForConfigEntry;
import org.appwork.storage.config.annotations.SpinnerValidator;
import org.jdownloader.plugins.config.Order;
@ -9,7 +10,7 @@ import org.jdownloader.plugins.config.PluginConfigInterface;
import org.jdownloader.plugins.config.PluginHost;
import org.jdownloader.plugins.config.Type;
@PluginHost(host = "civitai.com", type = Type.HOSTER)
@PluginHost(host = "civitai.com", type = Type.CRAWLER)
public interface CivitaiComConfig extends PluginConfigInterface {
public static final TRANSLATION TRANSLATION = new TRANSLATION();
@ -27,6 +28,7 @@ public interface CivitaiComConfig extends PluginConfigInterface {
@SpinnerValidator(min = 10, max = 200, step = 1)
@DefaultIntValue(50)
@DescriptionForConfigEntry("Internal value to limit max number of items per page. Lower value = More requests needed to crawl a profile. See: https://github.com/civitai/civitai/wiki/REST-API-Reference#get-apiv1images")
@DefaultOnNull()
@Order(10)
int getProfileCrawlerMaxPaginationItems();
@ -36,6 +38,7 @@ public interface CivitaiComConfig extends PluginConfigInterface {
@SpinnerValidator(min = 0, max = 20000, step = 1000)
@DefaultIntValue(1000)
@DescriptionForConfigEntry("Wait time between pagination requests.")
@DefaultOnNull()
@Order(20)
int getProfileCrawlerPaginationSleepMillis();

4
src/org/jdownloader/plugins/components/config/DiskYandexNetConfig.java
View File

@ -41,9 +41,9 @@ public interface DiskYandexNetConfig extends PluginConfigInterface {
@AboutConfig
@DefaultEnumValue("MOVE_INTO_ACCOUNT")
@Order(100)
@DescriptionForConfigEntry("What to do with quota limited files in account download mode?")
@DefaultOnNull
@Order(100)
ActionForQuotaLimitedFiles getActionForQuotaLimitedFiles();
void setActionForQuotaLimitedFiles(final ActionForQuotaLimitedFiles action);
@ -71,9 +71,9 @@ public interface DiskYandexNetConfig extends PluginConfigInterface {
@AboutConfig
@DefaultEnumValue("DELETE_FROM_ACCOUNT")
@Order(200)
@DescriptionForConfigEntry("What to do with files that JDownloader has moved/imported into your Yandex account?")
@DefaultOnNull
@Order(200)
ActionForMovedFiles getActionForMovedFiles();
void setActionForMovedFiles(final ActionForMovedFiles action);