mirror of
https://github.com/BillyOutlast/MidnightRider-Stash.git
synced 2026-10-12 02:25:09 +00:00
Remove previous markers and replace with new one which has TPDB and timestamp.trade markers
This commit is contained in:
Minasuki Hikimuna
+397
-93
@@ -24,100 +24,200 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tpdb_markers = stash.find_tag(\"Markers: TPDB\")\n",
|
||||
"tpdb_markers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ai_tagged = stash.find_tag(\"AI_Tagged\")\n",
|
||||
"ai_tagged"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"marker_source_tpdb = stash.find_tag(\"Marker Source: TPDB\")\n",
|
||||
"marker_source_tpdb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scenes_with_only_tpdb_markers = stash.find_scenes({\n",
|
||||
" \"tags\": {\n",
|
||||
" \"depth\": -1,\n",
|
||||
" \"value\": [tpdb_markers[\"id\"]],\n",
|
||||
" \"modifier\": \"INCLUDES_ALL\",\n",
|
||||
" \"excludes\": [ai_tagged[\"id\"]],\n",
|
||||
" }\n",
|
||||
"}, fragment=\"id title code details scene_markers { id title primary_tag { id name } tags { id name } }\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_scenes_with_only_tpdb_markers = pd.DataFrame(scenes_with_only_tpdb_markers)\n",
|
||||
"# Add a new column for the number of markers\n",
|
||||
"df_scenes_with_only_tpdb_markers['num_markers'] = df_scenes_with_only_tpdb_markers['scene_markers'].apply(lambda x: len(x) if isinstance(x, list) else 0)\n",
|
||||
"import os\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"df_scenes_with_only_tpdb_markers = df_scenes_with_only_tpdb_markers[df_scenes_with_only_tpdb_markers['num_markers'] > 1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for index, row in df_scenes_with_only_tpdb_markers.iterrows():\n",
|
||||
" # Check if all scene_markers have empty tags or only contain Marker Source: TPDB tag\n",
|
||||
" markers_empty = []\n",
|
||||
" markers_only_tpdb = []\n",
|
||||
" markers_invalid = []\n",
|
||||
"tpdb_headers = {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {os.environ['TPDB_API_KEY']}\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
" for marker in row['scene_markers']:\n",
|
||||
" if len(marker['tags']) == 0:\n",
|
||||
" markers_empty.append(marker)\n",
|
||||
" elif len(marker['tags']) == 1 and marker['tags'][0]['name'] == \"Marker Source: TPDB\":\n",
|
||||
" markers_only_tpdb.append(marker)\n",
|
||||
" else:\n",
|
||||
" markers_invalid.append(marker)\n",
|
||||
"def get_scene_tpdb_id(stash_scene):\n",
|
||||
" # Extract TPDB ID from stash_ids\n",
|
||||
" tpdb_id = None\n",
|
||||
" for stash_id in stash_scene.get('stash_ids', []):\n",
|
||||
" if stash_id['endpoint'] == 'https://theporndb.net/graphql':\n",
|
||||
" tpdb_id = stash_id['stash_id']\n",
|
||||
" break\n",
|
||||
" return tpdb_id\n",
|
||||
"\n",
|
||||
" all_markers_valid = len(markers_invalid) == 0\n",
|
||||
"def get_tpdb_scene_data(tpdb_id):\n",
|
||||
" url = f\"https://api.theporndb.net/scenes/{tpdb_id}\"\n",
|
||||
" response = requests.get(url, headers=tpdb_headers)\n",
|
||||
" return response.json()\n",
|
||||
"\n",
|
||||
" print(f\"All markers have empty tags or only Marker Source: TPDB tag: {all_markers_valid}\")\n",
|
||||
" print(f\"Markers with empty tags: {len(markers_empty)}\")\n",
|
||||
" print(f\"Markers with only Marker Source: TPDB tag: {len(markers_only_tpdb)}\")\n",
|
||||
"\n",
|
||||
" if markers_invalid:\n",
|
||||
" print(\"\\nMarkers with invalid tags:\")\n",
|
||||
" for marker in markers_invalid:\n",
|
||||
" print(f\"Marker ID: {marker['id']}, Title: {marker['title']}, Tags: {[tag['name'] for tag in marker['tags']]}\")\n",
|
||||
"# Function to convert seconds to MM:SS or HH:MM:SS format\n",
|
||||
"def format_time(seconds):\n",
|
||||
" minutes, secs = divmod(seconds, 60)\n",
|
||||
" hours, minutes = divmod(minutes, 60)\n",
|
||||
" if hours > 0:\n",
|
||||
" return f\"{hours:02d}:{minutes:02d}:{secs:02d}\"\n",
|
||||
" else:\n",
|
||||
" print(\"\\nNo markers found with invalid tags.\")\n",
|
||||
" return f\"{minutes:02d}:{secs:02d}\"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"marker_source_tpdb_tag = stash.find_tag(\"Marker Source: TPDB\")\n",
|
||||
"marker_source_skier_tag = stash.find_tag(\"Marker Source: Skier AI\")\n",
|
||||
"marker_source_manual_tag = stash.find_tag(\"Marker Source: Manual\")\n",
|
||||
"markers_tpdb_missing_tag = stash.find_tag(\"Markers: TPDB: Missing\")\n",
|
||||
"markers_tpdb_saved_tag = stash.find_tag(\"Markers: TPDB: Saved\")\n",
|
||||
"markers_timestamp_trade_missing_tag = stash.find_tag(\"Markers: timestamp.trade: Missing\")\n",
|
||||
"markers_timestamp_trade_saved_tag = stash.find_tag(\"Markers: timestamp.trade: Saved\")\n",
|
||||
"marker_source_timestamp_trade_tag = stash.find_tag(\"Marker Source: timestamp.trade\")\n",
|
||||
"\n",
|
||||
" # Update markers with empty tags to include Marker Source: TPDB tag\n",
|
||||
" for marker in markers_empty:\n",
|
||||
" stash.update_scene_marker({\n",
|
||||
" \"id\": marker[\"id\"],\n",
|
||||
" \"tag_ids\": [marker_source_tpdb[\"id\"]],\n",
|
||||
"# Check if all tags are truthy and print warnings if not\n",
|
||||
"tags_to_check = [\n",
|
||||
" marker_source_tpdb_tag,\n",
|
||||
" marker_source_skier_tag,\n",
|
||||
" marker_source_manual_tag,\n",
|
||||
" markers_tpdb_missing_tag,\n",
|
||||
" markers_tpdb_saved_tag,\n",
|
||||
" markers_timestamp_trade_missing_tag,\n",
|
||||
" markers_timestamp_trade_saved_tag,\n",
|
||||
" marker_source_timestamp_trade_tag\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for tag in tags_to_check:\n",
|
||||
" if not tag:\n",
|
||||
" print(f\"Warning: Tag '{tag['name']}' not found in Stash.\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Batch operation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scenes_with_tpdb_id = stash.find_scenes({\n",
|
||||
" \"stash_id_endpoint\": {\"endpoint\": \"https://theporndb.net/graphql\", \"modifier\": \"NOT_NULL\" },\n",
|
||||
" \"tags\": {\n",
|
||||
" \"value\": [],\n",
|
||||
" \"modifier\": \"INCLUDES_ALL\",\n",
|
||||
" \"excludes\": [markers_tpdb_missing_tag['id'], markers_tpdb_saved_tag['id']]\n",
|
||||
" }\n",
|
||||
"}, { \"per_page\": 250, \"page\": 1 })\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create markers from TPDB\n",
|
||||
"scenes_updated_with_markers = 0\n",
|
||||
"scenes_with_missing_markers = 0\n",
|
||||
"total_scenes_processed = 0\n",
|
||||
"tag_cache = {} # Cache for tag lookups\n",
|
||||
"\n",
|
||||
"def get_or_create_tag(title):\n",
|
||||
" if title in tag_cache:\n",
|
||||
" return tag_cache[title]\n",
|
||||
" \n",
|
||||
" tag = stash.find_tag(title)\n",
|
||||
" if not tag:\n",
|
||||
" # Retry with _TPDB suffix\n",
|
||||
" tpdb_title = f\"{title}_TPDB\"\n",
|
||||
" tag = stash.find_tag(tpdb_title)\n",
|
||||
" if not tag:\n",
|
||||
" # Create new tag with _TPDB suffix\n",
|
||||
" tag = stash.create_tag({\n",
|
||||
" \"name\": tpdb_title,\n",
|
||||
" \"description\": f\"Marker from TPDB. Original name: {title}\",\n",
|
||||
" })\n",
|
||||
" print(f\"Created tag '{tpdb_title}' in Stash\")\n",
|
||||
" \n",
|
||||
" tag_cache[title] = tag\n",
|
||||
" return tag\n",
|
||||
"\n",
|
||||
"for scene in scenes_with_tpdb_id:\n",
|
||||
" total_scenes_processed += 1\n",
|
||||
" tpdb_id = get_scene_tpdb_id(scene)\n",
|
||||
" tpdb_scene_data = get_tpdb_scene_data(tpdb_id)\n",
|
||||
" if 'data' not in tpdb_scene_data:\n",
|
||||
" print(f\"Error: 'data' key not found in TPDB scene data for scene ID: {scene['id']}\")\n",
|
||||
" continue\n",
|
||||
" tpdb_markers = tpdb_scene_data['data'].get('markers', [])\n",
|
||||
" scene_markers = scene['scene_markers']\n",
|
||||
"\n",
|
||||
" # Convert Stash markers to a set of (title, seconds) tuples for easy comparison\n",
|
||||
" stash_markers_set = set((marker['title'], marker['seconds']) for marker in scene_markers)\n",
|
||||
"\n",
|
||||
" # If TPDB does not have any markers, add markers_tpdb_missing_tag to the scene\n",
|
||||
" if not tpdb_markers:\n",
|
||||
" # Re-read the scene from Stash to get the most up-to-date data\n",
|
||||
" updated_scene = stash.find_scene(scene['id'])\n",
|
||||
" # Ensure we're using the most recent tag IDs\n",
|
||||
" current_tag_ids = [tag['id'] for tag in updated_scene['tags']]\n",
|
||||
" # Add the missing tag ID if it's not already present\n",
|
||||
" if markers_tpdb_missing_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_tpdb_missing_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene['id'],\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
" print(f\"Added 'Markers: TPDB: Missing' tag to scene ID: {scene['id']}, Title: {scene['title']}\")\n",
|
||||
" scenes_with_missing_markers += 1\n",
|
||||
" continue # Skip to the next scene\n",
|
||||
"\n",
|
||||
" print(f\"Updated {len(markers_empty)} markers with empty tags to include Marker Source: TPDB tag.\")\n"
|
||||
" # Find markers in TPDB that are not in Stash\n",
|
||||
" new_markers = []\n",
|
||||
" for tpdb_marker in tpdb_markers:\n",
|
||||
" title = tpdb_marker['title']\n",
|
||||
" seconds = tpdb_marker['start_time']\n",
|
||||
" if (title, seconds) not in stash_markers_set:\n",
|
||||
" tag = get_or_create_tag(title)\n",
|
||||
" tpdb_marker['tag_id'] = tag['id']\n",
|
||||
" new_markers.append(tpdb_marker)\n",
|
||||
"\n",
|
||||
" if new_markers:\n",
|
||||
" scenes_updated_with_markers += 1\n",
|
||||
" print(f\"Scene ID: {scene['id']}, Title: {scene['title']}, Total new markers to add: {len(new_markers)}\")\n",
|
||||
"\n",
|
||||
" for new_marker in new_markers:\n",
|
||||
" stash.create_scene_marker({\n",
|
||||
" \"scene_id\": scene['id'],\n",
|
||||
" \"title\": new_marker['title'],\n",
|
||||
" \"primary_tag_id\": new_marker['tag_id'],\n",
|
||||
" \"seconds\": new_marker['start_time'],\n",
|
||||
" \"tag_ids\": [marker_source_tpdb_tag['id']],\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" # Add 'Markers: TPDB: Saved' tag if there are TPDB markers and the scene is missing this tag\n",
|
||||
" if tpdb_markers:\n",
|
||||
" updated_scene = stash.find_scene(scene['id'])\n",
|
||||
" current_tag_ids = [tag['id'] for tag in updated_scene['tags']]\n",
|
||||
" if markers_tpdb_saved_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_tpdb_saved_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene['id'],\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
" print(f\"Added 'Markers: TPDB: Saved' tag to scene ID: {scene['id']}, Title: {scene['title']}\")\n",
|
||||
"\n",
|
||||
"print(f\"Total scenes processed: {total_scenes_processed}\")\n",
|
||||
"print(f\"Scenes updated with markers: {scenes_updated_with_markers}\")\n",
|
||||
"print(f\"Scenes with missing TPDB markers: {scenes_with_missing_markers}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# timestamp.trade"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -126,11 +226,50 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for marker in scene_markers:\n",
|
||||
" stash.update_scene_marker({\n",
|
||||
" \"id\": marker[\"id\"],\n",
|
||||
" \"tag_ids\": [marker_source_tpdb[\"id\"]],\n",
|
||||
" })"
|
||||
"# Read all scenes from Stash with unknown timestamp.trade status\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"def get_stashdb_id(scene):\n",
|
||||
" for stash_id in scene.get('stash_ids', []):\n",
|
||||
" if stash_id['endpoint'] == 'https://stashdb.org/graphql':\n",
|
||||
" return stash_id['stash_id']\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"def fetch_markers_from_timestamp_trade(stashdb_id):\n",
|
||||
" url = f\"https://timestamp.trade/get-markers/{stashdb_id}\"\n",
|
||||
" response = requests.get(url)\n",
|
||||
" if response.status_code == 200:\n",
|
||||
" result = response.json()\n",
|
||||
" return result if result != {} else None\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"# Read all scenes from Stash with stash_id_endpoint criteria\n",
|
||||
"all_scenes = []\n",
|
||||
"page = 1\n",
|
||||
"max_pages = 10\n",
|
||||
"per_page = 100 # Adjust this value based on your Stash instance's capabilities\n",
|
||||
"\n",
|
||||
"while True:\n",
|
||||
" scenes_batch = stash.find_scenes(\n",
|
||||
" {\n",
|
||||
" \"stash_id_endpoint\": {\"endpoint\": \"https://stashdb.org/graphql\", \"modifier\": \"NOT_NULL\"},\n",
|
||||
" \"tags\": {\n",
|
||||
" \"value\": [],\n",
|
||||
" \"modifier\": \"INCLUDES_ALL\",\n",
|
||||
" \"excludes\": [markers_timestamp_trade_missing_tag['id'], markers_timestamp_trade_saved_tag['id']]\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" filter={\"per_page\": per_page, \"page\": page}\n",
|
||||
" )\n",
|
||||
" if not scenes_batch:\n",
|
||||
" break\n",
|
||||
" all_scenes.extend(scenes_batch)\n",
|
||||
" page += 1\n",
|
||||
" print(f\"Read {len(all_scenes)} scenes from Stash\")\n",
|
||||
" if page > max_pages:\n",
|
||||
" break\n",
|
||||
"\n",
|
||||
"df_scenes_with_stashdb_id = pd.DataFrame(all_scenes)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -139,13 +278,178 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_scenes_with_only_tpdb_markers = pd.DataFrame(scenes_with_only_tpdb_markers)"
|
||||
"# Read metadata from timestamp.trade for all matching scenes\n",
|
||||
"from concurrent.futures import ThreadPoolExecutor\n",
|
||||
"\n",
|
||||
"def process_scene(index, scene):\n",
|
||||
" stashdb_id = get_stashdb_id(scene)\n",
|
||||
" markers = fetch_markers_from_timestamp_trade(stashdb_id)\n",
|
||||
" return index, markers\n",
|
||||
"\n",
|
||||
"df_scenes_with_stashdb_id['timestamp_trade_markers'] = None\n",
|
||||
"\n",
|
||||
"results = []\n",
|
||||
"\n",
|
||||
"with ThreadPoolExecutor(max_workers=10) as executor:\n",
|
||||
" futures = [executor.submit(process_scene, index, scene) for index, scene in df_scenes_with_stashdb_id.iterrows()]\n",
|
||||
" for i, future in enumerate(futures):\n",
|
||||
" index, markers = future.result()\n",
|
||||
" results.append((index, markers))\n",
|
||||
" \n",
|
||||
" # Print progress every 100 scenes\n",
|
||||
" if (i + 1) % 100 == 0:\n",
|
||||
" print(f\"Read markers from timestamp.trade for {i + 1} scenes\")\n",
|
||||
"\n",
|
||||
"# Update the DataFrame after all threads have completed\n",
|
||||
"for index, markers in results:\n",
|
||||
" df_scenes_with_stashdb_id.at[index, 'timestamp_trade_markers'] = markers\n",
|
||||
"\n",
|
||||
"# Create markers columns from timestamp.trade data\n",
|
||||
"df_scenes_with_stashdb_id['markers'] = df_scenes_with_stashdb_id['timestamp_trade_markers'].apply(lambda x: x.get('marker', []) if isinstance(x, dict) else [])\n",
|
||||
"df_scenes_with_stashdb_id['has_markers'] = df_scenes_with_stashdb_id['markers'].apply(lambda x: len(x) > 0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Add Markers: timestamp.trade: Missing tag to scenes with no markers\n",
|
||||
"df_scenes_without_markers = df_scenes_with_stashdb_id[~df_scenes_with_stashdb_id['has_markers']]\n",
|
||||
"\n",
|
||||
"for index, scene in df_scenes_without_markers.iterrows():\n",
|
||||
" scene_id = scene['id']\n",
|
||||
" updated_scene = stash.find_scene(scene_id)\n",
|
||||
" current_tag_ids = [tag['id'] for tag in updated_scene['tags']]\n",
|
||||
" if markers_timestamp_trade_missing_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_timestamp_trade_missing_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene_id,\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
" print(f\"Added 'Markers: timestamp.trade: Missing' tag to scene ID: {scene_id}, Title: {scene['title']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Match timestamp.trade markers to Stash tags\n",
|
||||
"\n",
|
||||
"# Create a DataFrame with distinct name and tag pairs\n",
|
||||
"distinct_pairs = set()\n",
|
||||
"\n",
|
||||
"for markers in df_scenes_with_stashdb_id['markers']:\n",
|
||||
" for marker in markers:\n",
|
||||
" name = marker.get('name', '')\n",
|
||||
" tag = marker.get('tag', '')\n",
|
||||
" distinct_pairs.add((name, tag))\n",
|
||||
"\n",
|
||||
"# Create a DataFrame with distinct name and tag pairs\n",
|
||||
"df_distinct_values = pd.DataFrame(list(distinct_pairs), columns=['name', 'tag'])\n",
|
||||
"\n",
|
||||
"# Sort the DataFrame for better readability\n",
|
||||
"df_distinct_values = df_distinct_values.sort_values(['name', 'tag']).reset_index(drop=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"df_distinct_values['matching_name_tags'] = None\n",
|
||||
"df_distinct_values['matching_tag_tags'] = None\n",
|
||||
"\n",
|
||||
"for index, row in df_distinct_values.iterrows():\n",
|
||||
" matching_name_tags = stash.find_tag(row['name']) if row['name'] else None\n",
|
||||
" matching_tag_tags = stash.find_tag(row['tag']) if row['tag'] else None\n",
|
||||
" df_distinct_values.at[index, 'matching_name_tags'] = matching_name_tags\n",
|
||||
" df_distinct_values.at[index, 'matching_tag_tags'] = matching_tag_tags\n",
|
||||
" \n",
|
||||
" # Print progress every 50 rows\n",
|
||||
" if (index + 1) % 50 == 0:\n",
|
||||
" print(f\"Processed {index + 1} rows\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Save timestamp.trade markers to Stash\n",
|
||||
"processed_scenes = 0\n",
|
||||
"for index, sample_row in df_scenes_with_stashdb_id[df_scenes_with_stashdb_id['has_markers']].iterrows():\n",
|
||||
" # Extract scene ID and markers\n",
|
||||
" scene_id = sample_row['id']\n",
|
||||
" markers = sample_row['markers']\n",
|
||||
" \n",
|
||||
" # Read scene data using stash.find_scene to verify that duplicates won't be created\n",
|
||||
" existing_scene = stash.find_scene(scene_id)\n",
|
||||
" \n",
|
||||
" # Create a list to store the marker data structures\n",
|
||||
" marker_data = []\n",
|
||||
" \n",
|
||||
" for marker in markers:\n",
|
||||
" name = marker.get('name', '')\n",
|
||||
" tag = marker.get('tag', '')\n",
|
||||
" start_time = marker.get('start', 0) / 1000.0\n",
|
||||
" \n",
|
||||
" # Find the corresponding row in df_distinct_values\n",
|
||||
" matching_row = df_distinct_values[(df_distinct_values['name'] == name) & (df_distinct_values['tag'] == tag)]\n",
|
||||
" \n",
|
||||
" if not matching_row.empty:\n",
|
||||
" # Get the matching tag ID from either name or tag column\n",
|
||||
" matching_tag = matching_row['matching_name_tags'].iloc[0] or matching_row['matching_tag_tags'].iloc[0]\n",
|
||||
" \n",
|
||||
" if matching_tag:\n",
|
||||
" # Create the marker data structure\n",
|
||||
" marker_data.append({\n",
|
||||
" 'scene_id': scene_id,\n",
|
||||
" 'tag_id': matching_tag['id'],\n",
|
||||
" 'tag_name': matching_tag['name'],\n",
|
||||
" 'start_time': start_time\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" for data in marker_data:\n",
|
||||
" # Check if the marker already exists in the scene\n",
|
||||
" marker_exists = False\n",
|
||||
" for marker in existing_scene['scene_markers']:\n",
|
||||
" if marker['primary_tag']['id'] == data['tag_id'] and abs(marker['seconds'] - data['start_time']) <= 5:\n",
|
||||
" marker_exists = True\n",
|
||||
" if marker_source_timestamp_trade_tag['id'] not in [tag['id'] for tag in marker['tags']]:\n",
|
||||
" updated_tags = [tag['id'] for tag in marker['tags']] + [marker_source_timestamp_trade_tag['id']]\n",
|
||||
" stash.update_scene_marker({\n",
|
||||
" \"id\": marker['id'],\n",
|
||||
" \"tag_ids\": updated_tags,\n",
|
||||
" })\n",
|
||||
" print(f\"Updated marker: {data['tag_name']} for scene ID: {data['scene_id']} with new tag\")\n",
|
||||
" break\n",
|
||||
" if not marker_exists:\n",
|
||||
" stash.create_scene_marker({\n",
|
||||
" \"scene_id\": data['scene_id'],\n",
|
||||
" \"title\": data['tag_name'],\n",
|
||||
" \"primary_tag_id\": data['tag_id'],\n",
|
||||
" \"seconds\": data['start_time'],\n",
|
||||
" \"tag_ids\": [marker_source_timestamp_trade_tag['id']],\n",
|
||||
" })\n",
|
||||
" print(f\"Created marker: {data['tag_name']} for scene ID: {data['scene_id']}\")\n",
|
||||
" \n",
|
||||
" refreshed_scene = stash.find_scene(scene_id)\n",
|
||||
" current_tag_ids = [tag['id'] for tag in refreshed_scene['tags']]\n",
|
||||
" if markers_timestamp_trade_saved_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_timestamp_trade_saved_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene_id,\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
" print(f\"Added 'Markers: Timestamp.trade: Saved' tag to scene ID: {scene_id}, Title: {existing_scene['title']}\")\n",
|
||||
" \n",
|
||||
" processed_scenes += 1\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "pandas",
|
||||
"display_name": "name_of_my_env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
|
||||
@@ -1,362 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create Stash app client\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"import dotenv\n",
|
||||
"\n",
|
||||
"from libraries.client_stashapp import get_stashapp_client\n",
|
||||
"\n",
|
||||
"dotenv.load_dotenv()\n",
|
||||
"\n",
|
||||
"stash = get_stashapp_client()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"tpdb_headers = {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {os.environ['TPDB_API_KEY']}\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"def get_scene_tpdb_id(stash_scene):\n",
|
||||
" # Extract TPDB ID from stash_ids\n",
|
||||
" tpdb_id = None\n",
|
||||
" for stash_id in stash_scene.get('stash_ids', []):\n",
|
||||
" if stash_id['endpoint'] == 'https://theporndb.net/graphql':\n",
|
||||
" tpdb_id = stash_id['stash_id']\n",
|
||||
" break\n",
|
||||
" return tpdb_id\n",
|
||||
"\n",
|
||||
"def get_tpdb_scene_data(tpdb_id):\n",
|
||||
" url = f\"https://api.theporndb.net/scenes/{tpdb_id}\"\n",
|
||||
" response = requests.get(url, headers=tpdb_headers)\n",
|
||||
" return response.json()\n",
|
||||
"\n",
|
||||
"# Function to convert seconds to MM:SS or HH:MM:SS format\n",
|
||||
"def format_time(seconds):\n",
|
||||
" minutes, secs = divmod(seconds, 60)\n",
|
||||
" hours, minutes = divmod(minutes, 60)\n",
|
||||
" if hours > 0:\n",
|
||||
" return f\"{hours:02d}:{minutes:02d}:{secs:02d}\"\n",
|
||||
" else:\n",
|
||||
" return f\"{minutes:02d}:{secs:02d}\"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"marker_source_tpdb = \"Marker Source: TPDB\"\n",
|
||||
"marker_source_skier = \"Marker Source: Skier AI\"\n",
|
||||
"marker_source_manual = \"Marker Source: Manual\"\n",
|
||||
"\n",
|
||||
"marker_source_tpdb_tag = stash.find_tag(marker_source_tpdb)\n",
|
||||
"marker_source_skier_tag = stash.find_tag(marker_source_skier)\n",
|
||||
"marker_source_manual_tag = stash.find_tag(marker_source_manual)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Processing single scene"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scene_id = 12661\n",
|
||||
"\n",
|
||||
"stash_scene = stash.find_scene(scene_id)\n",
|
||||
"print(stash_scene['title'])\n",
|
||||
"\n",
|
||||
"tpdb_id = get_scene_tpdb_id(stash_scene)\n",
|
||||
"print(tpdb_id)\n",
|
||||
"\n",
|
||||
"tpdb_scene_data = get_tpdb_scene_data(tpdb_id)\n",
|
||||
"print(tpdb_scene_data['data']['title'])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tpdb_markers = tpdb_scene_data['data']['markers']\n",
|
||||
"tpdb_markers\n",
|
||||
"\n",
|
||||
"for marker in tpdb_markers:\n",
|
||||
" print(marker)\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scene_markers = stash_scene['scene_markers']\n",
|
||||
"\n",
|
||||
"for marker in scene_markers:\n",
|
||||
" print({\n",
|
||||
" 'title': marker['title'],\n",
|
||||
" 'seconds': marker['seconds'],\n",
|
||||
" 'primary_tag': marker['primary_tag']['name'],\n",
|
||||
" 'tags': [tag['name'] for tag in marker['tags']]\n",
|
||||
" })"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Convert Stash markers to a set of (title, seconds) tuples for easy comparison\n",
|
||||
"stash_markers_set = set((marker['title'], marker['seconds']) for marker in scene_markers)\n",
|
||||
"\n",
|
||||
"# Function to convert seconds to MM:SS or HH:MM:SS format\n",
|
||||
"def format_time(seconds):\n",
|
||||
" minutes, secs = divmod(seconds, 60)\n",
|
||||
" hours, minutes = divmod(minutes, 60)\n",
|
||||
" if hours > 0:\n",
|
||||
" return f\"{hours:02d}:{minutes:02d}:{secs:02d}\"\n",
|
||||
" else:\n",
|
||||
" return f\"{minutes:02d}:{secs:02d}\"\n",
|
||||
"\n",
|
||||
"# Find markers in TPDB that are not in Stash\n",
|
||||
"new_markers = []\n",
|
||||
"print(\"Markers in TPDB but not in Stash:\")\n",
|
||||
"for tpdb_marker in tpdb_markers:\n",
|
||||
" title = tpdb_marker['title']\n",
|
||||
" seconds = tpdb_marker['start_time']\n",
|
||||
" formatted_time = format_time(seconds)\n",
|
||||
" if (title, seconds) not in stash_markers_set:\n",
|
||||
" print(f\"- {title} at {formatted_time}\")\n",
|
||||
" # Get the tag ID from Stash\n",
|
||||
" tag = stash.find_tag(title)\n",
|
||||
" if tag:\n",
|
||||
" tpdb_marker['tag_id'] = tag['id']\n",
|
||||
" else:\n",
|
||||
" print(f\"Warning: Tag '{title}' not found in Stash\")\n",
|
||||
" new_markers.append(tpdb_marker)\n",
|
||||
"\n",
|
||||
"print(f\"\\nTotal new markers to add: {len(new_markers)}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for new_marker in new_markers:\n",
|
||||
" stash.create_scene_marker({\n",
|
||||
" \"scene_id\": scene_id,\n",
|
||||
" \"title\": new_marker['title'],\n",
|
||||
" \"primary_tag_id\": new_marker['tag_id'],\n",
|
||||
" \"seconds\": new_marker['start_time'],\n",
|
||||
" \"tag_ids\": [marker_source_tpdb_tag['id']],\n",
|
||||
" })\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Batch operation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"markers_tpdb_missing_tag = stash.find_tag(\"Markers: TPDB: Missing\")\n",
|
||||
"markers_tpdb_missing_tag"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"markers_tpdb_saved_tag = stash.find_tag(\"Markers: TPDB: Saved\")\n",
|
||||
"markers_tpdb_saved_tag"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scenes_with_tpdb_id = stash.find_scenes({\n",
|
||||
" \"stash_id_endpoint\": {\"endpoint\": \"https://theporndb.net/graphql\", \"modifier\": \"NOT_NULL\" },\n",
|
||||
" \"tags\": {\n",
|
||||
" \"value\": [],\n",
|
||||
" \"modifier\": \"INCLUDES_ALL\",\n",
|
||||
" \"excludes\": [markers_tpdb_missing_tag['id'], markers_tpdb_saved_tag['id']]\n",
|
||||
" }\n",
|
||||
"}, { \"per_page\": 250, \"page\": 1 })\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scenes_updated_with_markers = 0\n",
|
||||
"for scene in scenes_with_tpdb_id:\n",
|
||||
" tpdb_id = get_scene_tpdb_id(scene)\n",
|
||||
" tpdb_scene_data = get_tpdb_scene_data(tpdb_id)\n",
|
||||
" tpdb_markers = tpdb_scene_data['data']['markers']\n",
|
||||
" scene_markers = scene['scene_markers']\n",
|
||||
"\n",
|
||||
" # Convert Stash markers to a set of (title, seconds) tuples for easy comparison\n",
|
||||
" stash_markers_set = set((marker['title'], marker['seconds']) for marker in scene_markers)\n",
|
||||
"\n",
|
||||
" # If TPDB does not have any markers, add markers_tpdb_missing_tag to the scene\n",
|
||||
" if not tpdb_markers:\n",
|
||||
" # Re-read the scene from Stash to get the most up-to-date data\n",
|
||||
" updated_scene = stash.find_scene(scene['id'])\n",
|
||||
" # Ensure we're using the most recent tag IDs\n",
|
||||
" current_tag_ids = [tag['id'] for tag in updated_scene['tags']]\n",
|
||||
" # Add the missing tag ID if it's not already present\n",
|
||||
" if markers_tpdb_missing_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_tpdb_missing_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene['id'],\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
" print(f\"Added 'Markers: TPDB: Missing' tag to scene ID: {scene['id']}, Title: {scene['title']}\")\n",
|
||||
" continue # Skip to the next scene\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # Find markers in TPDB that are not in Stash\n",
|
||||
" new_markers = []\n",
|
||||
" for tpdb_marker in tpdb_markers:\n",
|
||||
" title = tpdb_marker['title']\n",
|
||||
" seconds = tpdb_marker['start_time']\n",
|
||||
" if (title, seconds) not in stash_markers_set:\n",
|
||||
" # Get the tag ID from Stash\n",
|
||||
" tag = stash.find_tag(title)\n",
|
||||
" if tag:\n",
|
||||
" tpdb_marker['tag_id'] = tag['id']\n",
|
||||
" new_markers.append(tpdb_marker)\n",
|
||||
" else:\n",
|
||||
" print(f\"Warning: Tag '{title}' not found in Stash\")\n",
|
||||
"\n",
|
||||
" if new_markers:\n",
|
||||
" scenes_updated_with_markers += 1\n",
|
||||
" print(f\"Scene ID: {scene['id']}, Title: {scene['title']}, Total new markers to add: {len(new_markers)}\")\n",
|
||||
"\n",
|
||||
" for new_marker in new_markers:\n",
|
||||
" stash.create_scene_marker({\n",
|
||||
" \"scene_id\": scene['id'],\n",
|
||||
" \"title\": new_marker['title'],\n",
|
||||
" \"primary_tag_id\": new_marker['tag_id'],\n",
|
||||
" \"seconds\": new_marker['start_time'],\n",
|
||||
" \"tag_ids\": [marker_source_tpdb_tag['id']],\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" updated_scene = stash.find_scene(scene['id'])\n",
|
||||
" # Ensure we're using the most recent tag IDs\n",
|
||||
" current_tag_ids = [tag['id'] for tag in updated_scene['tags']]\n",
|
||||
" # Add the saved tag ID if it's not already present\n",
|
||||
" if markers_tpdb_saved_tag['id'] not in current_tag_ids:\n",
|
||||
" current_tag_ids.append(markers_tpdb_saved_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene['id'],\n",
|
||||
" \"tag_ids\": current_tag_ids\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Scenes updated with markers: {scenes_updated_with_markers}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Foo"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"all_tpdb_scene_markers = stash.find_scene_markers({\n",
|
||||
" \"tags\": {\n",
|
||||
" \"value\": [marker_source_tpdb_tag['id']],\n",
|
||||
" \"modifier\": \"INCLUDES_ALL\"\n",
|
||||
" }\n",
|
||||
"})\n",
|
||||
"\n",
|
||||
"unique_scene_ids = set()\n",
|
||||
"for marker in all_tpdb_scene_markers:\n",
|
||||
" unique_scene_ids.add(marker['scene']['id'])\n",
|
||||
"\n",
|
||||
"for scene_id in unique_scene_ids:\n",
|
||||
" scene = stash.find_scene(scene_id)\n",
|
||||
" tags = scene['tags']\n",
|
||||
" if not any(tag['id'] == markers_tpdb_missing_tag['id'] for tag in tags):\n",
|
||||
" update_tag_ids = [tag['id'] for tag in tags]\n",
|
||||
" update_tag_ids.append(markers_tpdb_saved_tag['id'])\n",
|
||||
" stash.update_scene({\n",
|
||||
" \"id\": scene_id,\n",
|
||||
" \"tag_ids\": update_tag_ids\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
" \n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "name_of_my_env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user