[HashTheStash] concurrent hashing, reuse code

.gitignore

@@ -2,4 +2,5 @@ renamer.log
 node_modules/
 _site/
 stashaid/app.log
-__pycache__/
+__pycache__/
+.env

plugins/HashTheStash/.env

@@ -1 +0,0 @@
- FAKE_INPUT={"server_connection": {"Scheme": "http", "Host": "0.0.0.0", "Port": 6969, "SessionCookie": {"Name": "session", "Value": "MTczMDE5NDYwMHxEWDhFQVFMX2dBQUJFQUVRQUFCc180QUFBZ1p6ZEhKcGJtY01DQUFHZFhObGNrbEVCbk4wY21sdVp3d0hBQVZ6ZEdGemFBWnpkSEpwYm1jTUZRQVRkbWx6YVhSbFpGQnNkV2RwYm5OSWIyOXJjeHRiWFhObGMzTnBiMjR1Vm1semFYUmxaRkJzZFdkcGJraHZiMnZfZ3dJQkF2LUVBQUhfZ2dBQU9mLUJBd0VCRVZacGMybDBaV1JRYkhWbmFXNUliMjlyQWYtQ0FBRUNBUWhRYkhWbmFXNUpSQUVNQUFFSVNHOXZhMVI1Y0dVQkRBQUFBQVhfaEFJQUFBPT188sCZj1eCf6a3DNGjj1ICXAAzBYznEUiHQP2SJQ2URFk=", "Quoted": false, "Path": "", "Domain": "", "Expires": "0001-01-01T00:00:00Z", "RawExpires": "", "MaxAge": 0, "Secure": false, "HttpOnly": false, "SameSite": 0, "Partitioned": false, "Raw": "", "Unparsed": null}, "Dir": "H:\\Stash", "PluginDir": "H:\\Stash\\plugins\\HashTheStash"}, "args": {"mode": "calculate_hashes"}}

plugins/HashTheStash/HashTheStash.py

@@ -2,52 +2,104 @@ import json
 import os
 import sys
 import hashlib
-import xxhash
 import stashapi.log as logger
 from stashapi.stashapp import StashInterface
+try:
+    import xxhash
+except ImportError:
+    logger.error("xxhash is not installed. Please install it using 'pip install xxhash'.")
+    has_xxhash = False
 
+def hash_file(file_path, add_xxhash=False, add_sha256=False, add_sha1=False):
+    fingerprints = []
+    if has_xxhash:
+        xxhash_hasher = xxhash.xxh64()
+    sha256_hasher = hashlib.sha256()
+    sha1_hasher = hashlib.sha1()
 
-def get_xxhash_of_file(file_path):
-    # Create an xxhash object
-    hasher = xxhash.xxh64()
-
-    # Open the file in binary mode
     with open(file_path, "rb") as f:
-        # Read and update hash string value in blocks of 4K
         for chunk in iter(lambda: f.read(4096), b""):
-            hasher.update(chunk)
+            if add_xxhash and has_xxhash:
+                xxhash_hasher.update(chunk)
+            if add_sha256:
+                sha256_hasher.update(chunk)
+            if add_sha1:
+                sha1_hasher.update(chunk)
 
-    # Return the hexadecimal representation of the hash
-    return hasher.hexdigest()
+    if add_xxhash and has_xxhash:
+        fingerprints.append({"type": "xxhash", "value": xxhash_hasher.hexdigest()})
+    if add_sha256:
+        fingerprints.append({"type": "sha256", "value": sha256_hasher.hexdigest()})
+    if add_sha1:
+        fingerprints.append({"type": "sha1", "value": sha1_hasher.hexdigest()})
+    return fingerprints
 
+def test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1):
+    file_path = file["path"]
+    if not os.path.exists(file_path):
+        logger.warning(f"File not found: {file_path}")
+        return
 
-def get_sha256_of_file(file_path):
-    # Create a SHA-256 hash object
-    hasher = hashlib.sha256()
+    add_xxhash = use_xxhash and ("fingerprints" not in file or not any(
+        fp["type"] == "xxhash" for fp in file["fingerprints"]
+    ))
+    add_sha256 = use_sha256 and ("fingerprints" not in file or not any(
+        fp["type"] == "sha256" for fp in file["fingerprints"]
+    ))
+    add_sha1 = use_sha1 and ("fingerprints" not in file or not any(
+        fp["type"] == "sha1" for fp in file["fingerprints"]
+    ))
 
-    # Open the file in binary mode
-    with open(file_path, "rb") as f:
-        # Read and update hash string value in blocks of 4K
-        for chunk in iter(lambda: f.read(4096), b""):
-            hasher.update(chunk)
+    if not add_xxhash and not add_sha256 and not add_sha1:
+        # if no fingerprints to add, just skip
+        return
 
-    # Return the hexadecimal representation of the hash
-    return hasher.hexdigest()
+    fingerprints_to_set = hash_file(file_path, add_xxhash, add_sha256, add_sha1)
 
+    if fingerprints_to_set:
+        logger.info(
+            f"File: {file_path}, new fingerprints: {fingerprints_to_set}"
+        )
+        stash.file_set_fingerprints(file["id"], fingerprints_to_set)
 
-def get_sha1_of_file(file_path):
-    # Create a SHA-1 hash object
-    hasher = hashlib.sha1()
+# common queries
+fragment = """
+    id
+    files {
+        id path
+        fingerprints {
+            type
+        }
+    }
+"""
 
-    # Open the file in binary mode
-    with open(file_path, "rb") as f:
-        # Read and update hash string value in blocks of 4K
-        for chunk in iter(lambda: f.read(4096), b""):
-            hasher.update(chunk)
+def hash_scenes(stash, use_xxhash, use_sha256, use_sha1):
+    scenes_result = stash.find_scenes(fragment=fragment, get_count=True)
+    scenes_count = scenes_result[0]
+    scenes = scenes_result[1]
 
-    # Return the hexadecimal representation of the hash
-    return hasher.hexdigest()
+    processed = 0
 
+    for scene in scenes:
+        for file in scene["files"]:
+            test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1)
+            processed += 1
+            logger.progress(processed/scenes_count)
+    logger.info(f"Processed a total of {scenes_count} scenes.")
+
+def hash_galleries(stash, use_xxhash, use_sha256, use_sha1):
+    galleries_result = stash.find_galleries(fragment=fragment, get_count=True)
+    galleries_count = galleries_result[0]
+    galleries = galleries_result[1]
+
+    processed = 0
+
+    for gallery in galleries:
+        for file in gallery["files"]:
+            test_hash_file(stash, file, use_xxhash, use_sha256, use_sha1)
+            processed += 1
+            logger.progress(processed/galleries_count)
+    logger.info(f"Processed a total of {galleries_count} galleries.")
 
 def get_json_input():
     if os.getenv("ENABLE_DEV_MODE"):
@@ -62,151 +114,6 @@ def get_json_input():
         raw_input = sys.stdin.read()
         return json.loads(raw_input)
 
-
-def hash_scenes(stash, use_xxhash, use_sha256, use_sha1):
-    fragment = """
-            id
-            files {
-                id
-                path
-                fingerprints {
-                    type 
-                    value
-                }
-            }
-    """
-
-    per_page = 25
-    page = 1
-    total_scenes = 0
-
-    while True:
-        scenes_result = stash.find_scenes(
-            {}, {"per_page": per_page, "page": page}, "", fragment, get_count=True
-        )
-        scenes_count = scenes_result[0]
-        scenes = scenes_result[1]
-
-        if not scenes:
-            break
-
-        total_scenes += len(scenes)
-
-        for scene in scenes:
-            for file in scene["files"]:
-                file_path = file["path"]
-                if not os.path.exists(file_path):
-                    logger.warning(f"File not found: {file_path}")
-                    continue
-
-                fingerprints_to_set = []
-
-                if use_xxhash and ("fingerprints" not in file or not any(
-                    fp["type"] == "xxhash" for fp in file["fingerprints"]
-                )):
-                    xxhash_value = get_xxhash_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "xxhash", "value": xxhash_value}
-                    )
-
-                if use_sha256 and ("fingerprints" not in file or not any(
-                    fp["type"] == "sha256" for fp in file["fingerprints"]
-                )):
-                    sha256_value = get_sha256_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "sha256", "value": sha256_value}
-                    )
-
-                if use_sha1 and ("fingerprints" not in file or not any(
-                    fp["type"] == "sha1" for fp in file["fingerprints"]
-                )):
-                    sha1_value = get_sha1_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "sha1", "value": sha1_value}
-                    )
-
-                if fingerprints_to_set:
-                    logger.info(
-                        f"File: {file_path}, new fingerprints: {fingerprints_to_set}"
-                    )
-                    stash.file_set_fingerprints(file["id"], fingerprints_to_set)
-
-        page += 1
-
-    logger.info(f"Processed a total of {total_scenes} scenes.")
-
-def hash_galleries(stash, use_xxhash, use_sha256, use_sha1):
-    fragment = """
-            id
-            files {
-                id
-                path
-                fingerprints {
-                    type 
-                    value
-                }
-            }
-    """
-
-    per_page = 25
-    page = 1
-    total_galleries = 0
-
-    while True:
-        galleries_result = stash.find_galleries(
-            {}, {"per_page": per_page, "page": page}, "", fragment, get_count=True
-        )
-        galleries_count = galleries_result[0]
-        galleries = galleries_result[1]
-
-        if not galleries:
-            break
-
-        total_galleries += len(galleries)
-
-        for gallery in galleries:
-            for file in gallery["files"]:
-                file_path = file["path"]
-                if not os.path.exists(file_path):
-                    logger.warning(f"File not found: {file_path}")
-                    continue
-
-                fingerprints_to_set = []
-
-                if use_xxhash and ("fingerprints" not in file or not any(
-                    fp["type"] == "xxhash" for fp in file["fingerprints"]
-                )):
-                    xxhash_value = get_xxhash_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "xxhash", "value": xxhash_value}
-                    )
-
-                if use_sha256 and ("fingerprints" not in file or not any(
-                    fp["type"] == "sha256" for fp in file["fingerprints"]
-                )):
-                    sha256_value = get_sha256_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "sha256", "value": sha256_value}
-                    )
-
-                if use_sha1 and ("fingerprints" not in file or not any(
-                    fp["type"] == "sha1" for fp in file["fingerprints"]
-                )):
-                    sha1_value = get_sha1_of_file(file_path)
-                    fingerprints_to_set.append(
-                        {"type": "sha1", "value": sha1_value}
-                    )
-
-                if fingerprints_to_set:
-                    logger.info(
-                        f"File: {file_path}, new fingerprints: {fingerprints_to_set}"
-                    )
-                    stash.file_set_fingerprints(file["id"], fingerprints_to_set)
-
-        page += 1
-
-    logger.info(f"Processed a total of {total_galleries} galleries.")
-
 # Example usage
 if __name__ == "__main__":
     json_input = get_json_input()
@@ -225,9 +132,13 @@ if __name__ == "__main__":
         logger.error("No hash type selected. Please select at least one hash type.")
         sys.exit(1)
 
+    if not has_xxhash and use_xxhash:
+        logger.error("xxhash is requested but not installed. Please install it using 'pip install xxhash'.")
+        sys.exit(1)
+
     use_hash_galleries = plugin_configuration.get("hash_galleries", False)
     use_hash_scenes = plugin_configuration.get("hash_scenes", False)
-    
+
     if not use_hash_scenes and not use_hash_galleries:
         logger.error("Neither scenes nor galleries selected for hashing. Please select at least one.")
         sys.exit(1)

plugins/HashTheStash/HashTheStash.yml

@@ -1,6 +1,6 @@
 name: HashTheStash
 description: Calculates additional fingerprints for scenes.
-version: 0.4.0
+version: 0.4.1
 url: https://github.com/MinasukiHikimuna/MidnightRider-Stash/
 exec:
   - python

plugins/HashTheStash/requirements.txt

@@ -0,0 +1 @@
+xxhash