diff --git a/.gitignore b/.gitignore index 2e65812..ae87ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ renamer.log node_modules/ _site/ stashaid/app.log -__pycache__/ \ No newline at end of file +__pycache__/ +.env \ No newline at end of file diff --git a/plugins/HashTheStash/.env b/plugins/HashTheStash/.env deleted file mode 100644 index 7ad3dcd..0000000 --- a/plugins/HashTheStash/.env +++ /dev/null @@ -1 +0,0 @@ - FAKE_INPUT={"server_connection": {"Scheme": "http", "Host": "0.0.0.0", "Port": 6969, "SessionCookie": {"Name": "session", "Value": "MTczMDE5NDYwMHxEWDhFQVFMX2dBQUJFQUVRQUFCc180QUFBZ1p6ZEhKcGJtY01DQUFHZFhObGNrbEVCbk4wY21sdVp3d0hBQVZ6ZEdGemFBWnpkSEpwYm1jTUZRQVRkbWx6YVhSbFpGQnNkV2RwYm5OSWIyOXJjeHRiWFhObGMzTnBiMjR1Vm1semFYUmxaRkJzZFdkcGJraHZiMnZfZ3dJQkF2LUVBQUhfZ2dBQU9mLUJBd0VCRVZacGMybDBaV1JRYkhWbmFXNUliMjlyQWYtQ0FBRUNBUWhRYkhWbmFXNUpSQUVNQUFFSVNHOXZhMVI1Y0dVQkRBQUFBQVhfaEFJQUFBPT188sCZj1eCf6a3DNGjj1ICXAAzBYznEUiHQP2SJQ2URFk=", "Quoted": false, "Path": "", "Domain": "", "Expires": "0001-01-01T00:00:00Z", "RawExpires": "", "MaxAge": 0, "Secure": false, "HttpOnly": false, "SameSite": 0, "Partitioned": false, "Raw": "", "Unparsed": null}, "Dir": "H:\\Stash", "PluginDir": "H:\\Stash\\plugins\\HashTheStash"}, "args": {"mode": "calculate_hashes"}} \ No newline at end of file diff --git a/plugins/HashTheStash/HashTheStash.py b/plugins/HashTheStash/HashTheStash.py index 3e1408d..65f24fc 100644 --- a/plugins/HashTheStash/HashTheStash.py +++ b/plugins/HashTheStash/HashTheStash.py @@ -2,52 +2,104 @@ import json import os import sys import hashlib -import xxhash import stashapi.log as logger from stashapi.stashapp import StashInterface +try: + import xxhash +except ImportError: + logger.error("xxhash is not installed. Please install it using 'pip install xxhash'.") + has_xxhash = False +def hash_file(file_path, add_xxhash=False, add_sha256=False, add_sha1=False): + fingerprints = [] + if has_xxhash: + xxhash_hasher = xxhash.xxh64() + sha256_hasher = hashlib.sha256() + sha1_hasher = hashlib.sha1() -def get_xxhash_of_file(file_path): - # Create an xxhash object - hasher = xxhash.xxh64() - - # Open the file in binary mode with open(file_path, "rb") as f: - # Read and update hash string value in blocks of 4K for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) + if add_xxhash and has_xxhash: + xxhash_hasher.update(chunk) + if add_sha256: + sha256_hasher.update(chunk) + if add_sha1: + sha1_hasher.update(chunk) - # Return the hexadecimal representation of the hash - return hasher.hexdigest() + if add_xxhash and has_xxhash: + fingerprints.append({"type": "xxhash", "value": xxhash_hasher.hexdigest()}) + if add_sha256: + fingerprints.append({"type": "sha256", "value": sha256_hasher.hexdigest()}) + if add_sha1: + fingerprints.append({"type": "sha1", "value": sha1_hasher.hexdigest()}) + return fingerprints +def test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1): + file_path = file["path"] + if not os.path.exists(file_path): + logger.warning(f"File not found: {file_path}") + return -def get_sha256_of_file(file_path): - # Create a SHA-256 hash object - hasher = hashlib.sha256() + add_xxhash = use_xxhash and ("fingerprints" not in file or not any( + fp["type"] == "xxhash" for fp in file["fingerprints"] + )) + add_sha256 = use_sha256 and ("fingerprints" not in file or not any( + fp["type"] == "sha256" for fp in file["fingerprints"] + )) + add_sha1 = use_sha1 and ("fingerprints" not in file or not any( + fp["type"] == "sha1" for fp in file["fingerprints"] + )) - # Open the file in binary mode - with open(file_path, "rb") as f: - # Read and update hash string value in blocks of 4K - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) + if not add_xxhash and not add_sha256 and not add_sha1: + # if no fingerprints to add, just skip + return - # Return the hexadecimal representation of the hash - return hasher.hexdigest() + fingerprints_to_set = hash_file(file_path, add_xxhash, add_sha256, add_sha1) + if fingerprints_to_set: + logger.info( + f"File: {file_path}, new fingerprints: {fingerprints_to_set}" + ) + stash.file_set_fingerprints(file["id"], fingerprints_to_set) -def get_sha1_of_file(file_path): - # Create a SHA-1 hash object - hasher = hashlib.sha1() +# common queries +fragment = """ + id + files { + id path + fingerprints { + type + } + } +""" - # Open the file in binary mode - with open(file_path, "rb") as f: - # Read and update hash string value in blocks of 4K - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) +def hash_scenes(stash, use_xxhash, use_sha256, use_sha1): + scenes_result = stash.find_scenes(fragment=fragment, get_count=True) + scenes_count = scenes_result[0] + scenes = scenes_result[1] - # Return the hexadecimal representation of the hash - return hasher.hexdigest() + processed = 0 + for scene in scenes: + for file in scene["files"]: + test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1) + processed += 1 + logger.progress(processed/scenes_count) + logger.info(f"Processed a total of {scenes_count} scenes.") + +def hash_galleries(stash, use_xxhash, use_sha256, use_sha1): + galleries_result = stash.find_galleries(fragment=fragment, get_count=True) + galleries_count = galleries_result[0] + galleries = galleries_result[1] + + processed = 0 + + for gallery in galleries: + for file in gallery["files"]: + test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1) + processed += 1 + logger.progress(processed/galleries_count) + logger.info(f"Processed a total of {galleries_count} galleries.") def get_json_input(): if os.getenv("ENABLE_DEV_MODE"): @@ -62,151 +114,6 @@ def get_json_input(): raw_input = sys.stdin.read() return json.loads(raw_input) - -def hash_scenes(stash, use_xxhash, use_sha256, use_sha1): - fragment = """ - id - files { - id - path - fingerprints { - type - value - } - } - """ - - per_page = 25 - page = 1 - total_scenes = 0 - - while True: - scenes_result = stash.find_scenes( - {}, {"per_page": per_page, "page": page}, "", fragment, get_count=True - ) - scenes_count = scenes_result[0] - scenes = scenes_result[1] - - if not scenes: - break - - total_scenes += len(scenes) - - for scene in scenes: - for file in scene["files"]: - file_path = file["path"] - if not os.path.exists(file_path): - logger.warning(f"File not found: {file_path}") - continue - - fingerprints_to_set = [] - - if use_xxhash and ("fingerprints" not in file or not any( - fp["type"] == "xxhash" for fp in file["fingerprints"] - )): - xxhash_value = get_xxhash_of_file(file_path) - fingerprints_to_set.append( - {"type": "xxhash", "value": xxhash_value} - ) - - if use_sha256 and ("fingerprints" not in file or not any( - fp["type"] == "sha256" for fp in file["fingerprints"] - )): - sha256_value = get_sha256_of_file(file_path) - fingerprints_to_set.append( - {"type": "sha256", "value": sha256_value} - ) - - if use_sha1 and ("fingerprints" not in file or not any( - fp["type"] == "sha1" for fp in file["fingerprints"] - )): - sha1_value = get_sha1_of_file(file_path) - fingerprints_to_set.append( - {"type": "sha1", "value": sha1_value} - ) - - if fingerprints_to_set: - logger.info( - f"File: {file_path}, new fingerprints: {fingerprints_to_set}" - ) - stash.file_set_fingerprints(file["id"], fingerprints_to_set) - - page += 1 - - logger.info(f"Processed a total of {total_scenes} scenes.") - -def hash_galleries(stash, use_xxhash, use_sha256, use_sha1): - fragment = """ - id - files { - id - path - fingerprints { - type - value - } - } - """ - - per_page = 25 - page = 1 - total_galleries = 0 - - while True: - galleries_result = stash.find_galleries( - {}, {"per_page": per_page, "page": page}, "", fragment, get_count=True - ) - galleries_count = galleries_result[0] - galleries = galleries_result[1] - - if not galleries: - break - - total_galleries += len(galleries) - - for gallery in galleries: - for file in gallery["files"]: - file_path = file["path"] - if not os.path.exists(file_path): - logger.warning(f"File not found: {file_path}") - continue - - fingerprints_to_set = [] - - if use_xxhash and ("fingerprints" not in file or not any( - fp["type"] == "xxhash" for fp in file["fingerprints"] - )): - xxhash_value = get_xxhash_of_file(file_path) - fingerprints_to_set.append( - {"type": "xxhash", "value": xxhash_value} - ) - - if use_sha256 and ("fingerprints" not in file or not any( - fp["type"] == "sha256" for fp in file["fingerprints"] - )): - sha256_value = get_sha256_of_file(file_path) - fingerprints_to_set.append( - {"type": "sha256", "value": sha256_value} - ) - - if use_sha1 and ("fingerprints" not in file or not any( - fp["type"] == "sha1" for fp in file["fingerprints"] - )): - sha1_value = get_sha1_of_file(file_path) - fingerprints_to_set.append( - {"type": "sha1", "value": sha1_value} - ) - - if fingerprints_to_set: - logger.info( - f"File: {file_path}, new fingerprints: {fingerprints_to_set}" - ) - stash.file_set_fingerprints(file["id"], fingerprints_to_set) - - page += 1 - - logger.info(f"Processed a total of {total_galleries} galleries.") - # Example usage if __name__ == "__main__": json_input = get_json_input() @@ -225,9 +132,13 @@ if __name__ == "__main__": logger.error("No hash type selected. Please select at least one hash type.") sys.exit(1) + if not has_xxhash and use_xxhash: + logger.error("xxhash is requested but not installed. Please install it using 'pip install xxhash'.") + sys.exit(1) + use_hash_galleries = plugin_configuration.get("hash_galleries", False) use_hash_scenes = plugin_configuration.get("hash_scenes", False) - + if not use_hash_scenes and not use_hash_galleries: logger.error("Neither scenes nor galleries selected for hashing. Please select at least one.") sys.exit(1) diff --git a/plugins/HashTheStash/HashTheStash.yml b/plugins/HashTheStash/HashTheStash.yml index 2283b8e..1bd54b8 100644 --- a/plugins/HashTheStash/HashTheStash.yml +++ b/plugins/HashTheStash/HashTheStash.yml @@ -1,6 +1,6 @@ name: HashTheStash description: Calculates additional fingerprints for scenes. -version: 0.4.0 +version: 0.4.1 url: https://github.com/MinasukiHikimuna/MidnightRider-Stash/ exec: - python diff --git a/plugins/HashTheStash/requirements.txt b/plugins/HashTheStash/requirements.txt new file mode 100644 index 0000000..ca5e064 --- /dev/null +++ b/plugins/HashTheStash/requirements.txt @@ -0,0 +1 @@ +xxhash \ No newline at end of file