jellyfin-kodi/resources/lib/librarysync.py
2018-06-09 00:20:15 -05:00

866 lines
32 KiB
Python

# -*- coding: utf-8 -*-
##################################################################################################
import logging
import sqlite3
import threading
from datetime import datetime, timedelta, time
import xbmc
import xbmcgui
import xbmcvfs
import api
import utils
import clientinfo
import database
import downloadutils
import itemtypes
import emby as mb
import embydb_functions as embydb
import read_embyserver as embyserver
import userclient
import views
from objects import Movies, MusicVideos, TVShows, Music
from utils import window, settings, language as lang, should_stop
#from ga_client import GoogleAnalytics
##################################################################################################
log = logging.getLogger("EMBY."+__name__)
##################################################################################################
class LibrarySync(threading.Thread):
_shared_state = {}
isFastSync = False
stop_thread = False
suspend_thread = False
# Track websocketclient updates
addedItems = []
updateItems = []
userdataItems = []
removeItems = []
forceLibraryUpdate = False
incremental_count = 0
refresh_views = False
def __init__(self):
self.__dict__ = self._shared_state
self.monitor = xbmc.Monitor()
self.clientInfo = clientinfo.ClientInfo()
self.doUtils = downloadutils.DownloadUtils().downloadUrl
self.user = userclient.UserClient()
self.emby = embyserver.Read_EmbyServer()
self.kodi_version = int(xbmc.getInfoLabel('System.BuildVersion')[:2])
threading.Thread.__init__(self)
def progressDialog(self, title):
dialog = None
dialog = xbmcgui.DialogProgressBG()
dialog.create("Emby for Kodi", title)
log.debug("Show progress dialog: %s" % title)
return dialog
def startSync(self):
#ga = GoogleAnalytics()
# Run at start up - optional to use the server plugin
if settings('SyncInstallRunDone') == "true":
# Validate views
self.refreshViews()
completed = False
# Verify if server plugin is installed.
if settings('serverSync') == "true":
# Try to use fast start up
url = "{server}/emby/Plugins?format=json"
try:
result = self.doUtils(url)
except Exception as error:
log.info("Error getting plugin list form server: " + str(error))
result = []
for plugin in result:
if plugin['Name'] in ("Emby.Kodi Sync Queue", "Kodi companion"):
log.debug("Found server plugin.")
self.isFastSync = True
#ga.sendEventData("SyncAction", "FastSync")
completed = self.fastSync()
break
if not completed:
# Fast sync failed or server plugin is not found
#ga.sendEventData("SyncAction", "Sync")
completed = ManualSync().sync()
else:
# Install sync is not completed
#ga.sendEventData("SyncAction", "FullSync")
completed = self.fullSync()
return completed
def fastSync(self):
lastSync = settings('LastIncrementalSync')
if not lastSync:
lastSync = "2010-01-01T00:00:00Z"
lastSyncTime = utils.convertDate(lastSync)
log.info("Last sync run: %s" % lastSyncTime)
# get server RetentionDateTime
try:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
retention_time = result['RetentionDateTime']
except Exception as error:
log.error(error)
retention_time = "2010-01-01T00:00:00Z"
retention_time = utils.convertDate(retention_time)
log.info("RetentionDateTime: %s" % retention_time)
# if last sync before retention time do a full sync
if retention_time > lastSyncTime:
log.info("Fast sync server retention insufficient, fall back to full sync")
return False
params = {'LastUpdateDT': lastSync}
if settings('enableMusic') != "true":
params['filter'] = "music"
url = "{server}/emby/Emby.Kodi.SyncQueue/{UserId}/GetItems?format=json"
try:
result = self.doUtils(url, parameters=params)
processlist = {
'added': result['ItemsAdded'],
'update': result['ItemsUpdated'],
'userdata': result['UserDataChanged'],
'remove': result['ItemsRemoved']
}
except Exception as error: # To be reviewed to only catch specific errors.
log.error(error)
log.error("Failed to retrieve latest updates using fast sync.")
xbmcgui.Dialog().ok(lang(29999), lang(33095))
return False
else:
log.info("Fast sync changes: %s" % result)
for action in processlist:
self.triage_items(action, processlist[action])
return True
def saveLastSync(self):
# Save last sync time
overlap = 2
try: # datetime fails when used more than once, TypeError
if self.isFastSync:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
server_time = result['ServerDateTime']
server_time = utils.convertDate(server_time)
else:
raise Exception("Fast sync server plugin is not enabled.")
except Exception as e:
# If the server plugin is not installed or an error happened.
log.debug("An exception occurred: %s" % e)
time_now = datetime.utcnow()-timedelta(minutes=overlap)
lastSync = time_now.strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: client time -%s min: %s" % (overlap, lastSync))
else:
lastSync = (server_time - timedelta(minutes=overlap)).strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: server time -%s min: %s" % (overlap, lastSync))
finally:
settings('LastIncrementalSync', value=lastSync)
def dbCommit(self, connection):
# Central commit, verifies if Kodi database update is running
kodidb_scan = window('emby_kodiScan') == "true"
count = 0
while kodidb_scan:
log.info("Kodi scan is running. Waiting...")
kodidb_scan = window('emby_kodiScan') == "true"
if count == 10:
log.info("Flag still active, but will try to commit")
window('emby_kodiScan', clear=True)
if should_stop():
log.info("Commit unsuccessful. Sync terminated.")
break
if self.monitor.waitForAbort(1):
# Abort was requested while waiting. We should exit
log.info("Commit unsuccessful.")
break
count += 1
try:
connection.commit()
log.info("Commit successful.")
except sqlite3.OperationalError as error:
log.error(error)
if "database is locked" in error:
log.info("retrying...")
window('emby_kodiScan', value="true")
self.dbCommit(connection)
def fullSync(self, manualrun=False, repair=False):
# Only run once when first setting up. Can be run manually.
music_enabled = settings('enableMusic') == "true"
xbmc.executebuiltin('InhibitIdleShutdown(true)')
screensaver = utils.getScreensaver()
utils.setScreensaver(value="")
window('emby_dbScan', value="true")
# Add sources
utils.sourcesXML()
# use emby and video DBs
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn('video') as cursor_video:
# content sync: movies, tvshows, musicvideos, music
if manualrun:
message = "Manual sync"
elif repair:
message = "Repair sync"
repair_list = []
choices = ['all', 'movies', 'musicvideos', 'tvshows']
if music_enabled:
choices.append('music')
if self.kodi_version > 15:
# Jarvis or higher
types = xbmcgui.Dialog().multiselect(lang(33094), choices)
if types is None:
pass
elif 0 in types: # all
choices.pop(0)
repair_list.extend(choices)
else:
for index in types:
repair_list.append(choices[index])
else:
resp = xbmcgui.Dialog().select(lang(33094), choices)
if resp == 0: # all
choices.pop(resp)
repair_list.extend(choices)
else:
repair_list.append(choices[resp])
log.info("Repair queued for: %s", repair_list)
else:
message = "Initial sync"
window('emby_initialScan', value="true")
pDialog = self.progressDialog("%s" % message)
starttotal = datetime.now()
# Set views
views.Views(cursor_emby, cursor_video).maintain()
cursor_emby.connection.commit()
#self.maintainViews(cursor_emby, cursor_video)
# Sync video library
process = {
'movies': self.movies,
'boxsets': self.boxsets,
'musicvideos': self.musicvideos,
'tvshows': self.tvshows
}
for itemtype in process:
if repair and itemtype not in repair_list:
continue
startTime = datetime.now()
completed = process[itemtype](cursor_emby, cursor_video, pDialog)
if not completed:
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
if pDialog:
pDialog.close()
return False
else:
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished %s in: %s)"
% (itemtype, str(elapsedTime).split('.')[0]))
# sync music
# use emby and music
if music_enabled:
if repair and 'music' not in repair_list:
pass
else:
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn('music') as cursor_music:
startTime = datetime.now()
completed = self.music(cursor_emby, cursor_music, pDialog)
if not completed:
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
if pDialog:
pDialog.close()
return False
else:
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished music in: %s)"
% (str(elapsedTime).split('.')[0]))
if pDialog:
pDialog.close()
with database.DatabaseConn('emby') as cursor_emby:
emby_db = embydb.Embydb_Functions(cursor_emby)
current_version = emby_db.get_version(self.clientInfo.get_version())
window('emby_version', current_version)
settings('SyncInstallRunDone', value="true")
self.saveLastSync()
xbmc.executebuiltin('UpdateLibrary(video)')
elapsedtotal = datetime.now() - starttotal
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
window('emby_initialScan', clear=True)
xbmcgui.Dialog().notification(
heading=lang(29999),
message="%s %s %s" %
(message, lang(33025), str(elapsedtotal).split('.')[0]),
icon="special://home/addons/plugin.video.emby/icon.png",
sound=False)
return True
def refreshViews(self):
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn() as cursor_video:
# Compare views, assign correct tags to items
views.Views(cursor_emby, cursor_video).maintain()
def offline_mode_views(self):
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn() as cursor_video:
views.Views(cursor_emby, cursor_video).offline_mode()
def movies(self, embycursor, kodicursor, pdialog):
# Get movies from emby
emby_db = embydb.Embydb_Functions(embycursor)
movies = Movies(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('movies')
views += emby_db.getView_byType('mixed')
log.info("Media folders: %s" % views)
##### PROCESS MOVIES #####
for view in views:
log.info("Processing: %s", view)
view_name = view['name']
# Get items per view
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33017), view_name))
movies.count = 0
for all_movies in mb.get_items(view['id'], "Movie"):
if should_stop():
return False
movies.add_all("Movie", all_movies, view)
log.debug("Movies finished.")
return True
def boxsets(self, embycursor, kodicursor, pdialog):
movies = Movies(embycursor, kodicursor, pdialog)
if pdialog:
pdialog.update(heading=lang(29999), message=lang(33018))
movies.count = 0
for boxsets in mb.get_items(None, "BoxSet"):
if should_stop():
return False
movies.add_all("BoxSet", boxsets)
log.debug("Boxsets finished.")
return True
def musicvideos(self, embycursor, kodicursor, pdialog):
# Get musicvideos from emby
emby_db = embydb.Embydb_Functions(embycursor)
mvideos = MusicVideos(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('musicvideos')
log.info("Media folders: %s" % views)
for view in views:
log.info("Processing: %s", view)
# Get items per view
viewName = view['name']
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33019), viewName))
# Initial or repair sync
mvideos.count = 0
for all_mvideos in mb.get_items(view['id'], "MusicVideo"):
if should_stop():
return False
mvideos.add_all("MusicVideo", all_mvideos, view)
else:
log.debug("MusicVideos finished.")
return True
def tvshows(self, embycursor, kodicursor, pdialog):
# Get shows from emby
emby_db = embydb.Embydb_Functions(embycursor)
tvshows = TVShows(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('tvshows')
views += emby_db.getView_byType('mixed')
log.info("Media folders: %s" % views)
for view in views:
# Get items per view
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33020), view['name']))
tvshows.count = 0
for all_tvshows in mb.get_items(view['id'], "Series"):
if should_stop():
return False
tvshows.add_all("Series", all_tvshows, view)
else:
log.debug("TVShows finished.")
return True
def music(self, embycursor, kodicursor, pdialog):
# Get music from emby
emby_db = embydb.Embydb_Functions(embycursor)
music = Music(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('music')
log.info("Media folders: %s", views)
# Add music artists and everything will fall into place
if pdialog:
pdialog.update(heading=lang(29999),
message="%s Music..." % lang(33021))
for view in views:
music.count = 0
for all_artists in mb.get_artists(view['id']):
if should_stop():
return False
music.add_all("MusicArtist", all_artists)
log.debug("Finished syncing music")
return True
# Reserved for websocket_client.py and fast start
def triage_items(self, process, items):
processlist = {
'added': self.addedItems,
'update': self.updateItems,
'userdata': self.userdataItems,
'remove': self.removeItems
}
if items:
if process == "userdata":
itemids = []
for item in items:
itemids.append(item['ItemId'])
items = itemids
log.info("Queue %s: %s" % (process, items))
processlist[process].extend(items)
def incrementalSync(self):
self.incremental_count += 1
update_embydb = False
pDialog = None
# do a view update if needed
if self.refresh_views:
self.refreshViews()
self.refresh_views = False
self.forceLibraryUpdate = True
# do a lib update if any items in list
totalUpdates = len(self.addedItems) + len(self.updateItems) + len(self.userdataItems) + len(self.removeItems)
if totalUpdates > 0 and window('emby_kodiScan') != "true":
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn('video') as cursor_video:
xbmc.executebuiltin('InhibitIdleShutdown(true)')
screensaver = utils.getScreensaver()
utils.setScreensaver(value="")
emby_db = embydb.Embydb_Functions(cursor_emby)
incSyncIndicator = int(settings('incSyncIndicator') or 10)
if incSyncIndicator != -1 and totalUpdates > incSyncIndicator:
# Only present dialog if we are going to process items
pDialog = self.progressDialog('Incremental sync')
log.info("incSyncIndicator=" + str(incSyncIndicator) + " totalUpdates=" + str(totalUpdates))
process = {
'added': self.addedItems,
'update': self.updateItems,
'userdata': self.userdataItems,
'remove': self.removeItems
}
for process_type in ['added', 'update', 'userdata', 'remove']:
if process[process_type]:
listItems = list(process[process_type])
del process[process_type][:] # Reset class list
items_process = itemtypes.Items(cursor_emby, cursor_video)
update = False
# Prepare items according to process process_type
if process_type == "added":
items = mb.sortby_mediatype(listItems)
elif process_type in ("userdata", "remove"):
items = emby_db.sortby_mediaType(listItems, unsorted=False)
else:
items = emby_db.sortby_mediaType(listItems)
if items.get('Unsorted'):
sorted_items = mb.sortby_mediatype(items['Unsorted'])
doupdate = items_process.itemsbyId(sorted_items, "added", pDialog)
if doupdate:
embyupdate, kodiupdate_video = doupdate
if embyupdate:
update_embydb = True
if kodiupdate_video:
self.forceLibraryUpdate = True
del items['Unsorted']
doupdate = items_process.itemsbyId(items, process_type, pDialog)
if doupdate:
embyupdate, kodiupdate_video = doupdate
if embyupdate:
update_embydb = True
if kodiupdate_video:
self.forceLibraryUpdate = True
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
# if stuff happened then do some stuff
if update_embydb:
update_embydb = False
log.info("Updating emby database.")
self.saveLastSync()
if self.forceLibraryUpdate:
# Force update the Kodi library
self.forceLibraryUpdate = False
log.info("Updating video library.")
self.incremental_count = 0
window('emby_kodiScan', value="true")
xbmc.executebuiltin('UpdateLibrary(video)')
if pDialog:
pDialog.close()
def compareDBVersion(self, current, minimum):
# It returns True is database is up to date. False otherwise.
log.info("current: %s minimum: %s" % (current, minimum))
try:
currMajor, currMinor, currPatch = current.split(".")
minMajor, minMinor, minPatch = minimum.split(".")
except ValueError as error:
raise ValueError("Unable to compare versions: %s, %s" % (current, minimum))
if currMajor > minMajor:
return True
elif currMajor == minMajor and (currMinor > minMinor or
(currMinor == minMinor and currPatch >= minPatch)):
return True
else:
# Database out of date.
return False
def run(self):
try:
self.run_internal()
except Warning as e:
if "restricted" in e:
pass
elif "401" in e:
pass
except Exception as e:
"""
ga = GoogleAnalytics()
errStrings = ga.formatException()
if not (hasattr(e, 'quiet') and e.quiet):
ga.sendEventData("Exception", errStrings[0], errStrings[1])
"""
window('emby_dbScan', clear=True)
log.exception(e)
xbmcgui.Dialog().ok(
heading=lang(29999),
line1=(
"Library sync thread has exited! "
"You should restart Kodi now. "
"Please report this on the forum."))
#line2=(errStrings[0] + " (" + errStrings[1] + ")"))
def run_internal(self):
dialog = xbmcgui.Dialog()
startupComplete = False
log.warn("---===### Starting LibrarySync ###===---")
if utils.verify_advancedsettings():
# Advancedsettings was modified, Kodi needs to restart
log.warn("###===--- LibrarySync Aborted ---===###")
return
while not self.monitor.abortRequested():
# In the event the server goes offline
while self.suspend_thread:
# Set in service.py
if self.monitor.waitForAbort(5):
# Abort was requested while waiting. We should exit
break
if (window('emby_dbCheck') != "true" and settings('SyncInstallRunDone') == "true"):
# Verify the validity of the database
log.info("Doing DB Version Check")
with database.DatabaseConn('emby') as cursor:
emby_db = embydb.Embydb_Functions(cursor)
currentVersion = emby_db.get_version()
###$ Begin migration $###
if not currentVersion:
currentVersion = emby_db.get_version(settings('dbCreatedWithVersion') or self.clientInfo.get_version())
log.info("Migration of database version completed")
###$ End migration $###
window('emby_version', value=currentVersion)
minVersion = window('emby_minDBVersion')
uptoDate = self.compareDBVersion(currentVersion, minVersion)
if not uptoDate:
log.warn("Database version out of date: %s minimum version required: %s"
% (currentVersion, minVersion))
resp = dialog.yesno(lang(29999), lang(33022))
if not resp:
log.warn("Database version is out of date! USER IGNORED!")
dialog.ok(lang(29999), lang(33023))
else:
database.db_reset()
break
window('emby_dbCheck', value="true")
if not startupComplete:
# Verify the video database can be found
videoDb = database.video_database()
if not xbmcvfs.exists(videoDb):
# Database does not exists
log.error(
"The current Kodi version is incompatible "
"with the Emby for Kodi add-on. Please visit "
"https://github.com/MediaBrowser/Emby.Kodi/wiki "
"to know which Kodi versions are supported.")
dialog.ok(
heading=lang(29999),
line1=lang(33024))
break
# Run start up sync
log.warn("Database version: %s", window('emby_version'))
log.info("SyncDatabase (started)")
startTime = datetime.now()
librarySync = self.startSync()
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished in: %s) %s"
% (str(elapsedTime).split('.')[0], librarySync))
# Add other servers at this point
# TODO: re-add once plugin listing is created
# self.user.load_connect_servers()
# Only try the initial sync once per kodi session regardless
# This will prevent an infinite loop in case something goes wrong.
startupComplete = True
window('emby.connected', value="true")
# Process updates
if self.incremental_count > 5:
self.incremental_count = 0
window('emby_kodiScan', clear=True)
if ((not xbmc.Player().isPlayingVideo() or xbmc.getCondVisibility('VideoPlayer.Content(livetv)')) and
window('emby_dbScan') != "true" and window('emby_shouldStop') != "true"):
self.incrementalSync()
if window('emby_onWake') == "true" and window('emby_online') == "true":
# Kodi is waking up
# Set in kodimonitor.py
window('emby_onWake', clear=True)
if window('emby_syncRunning') != "true":
log.info("SyncDatabase onWake (started)")
librarySync = self.startSync()
log.info("SyncDatabase onWake (finished) %s" % librarySync)
if self.stop_thread:
# Set in service.py
log.debug("Service terminated thread.")
break
if self.monitor.waitForAbort(1):
# Abort was requested while waiting. We should exit
break
log.warn("###===--- LibrarySync Stopped ---===###")
def stopThread(self):
self.stop_thread = True
log.debug("Ending thread...")
def suspendThread(self):
self.suspend_thread = True
log.debug("Pausing thread...")
def resumeThread(self):
self.suspend_thread = False
log.debug("Resuming thread...")
class ManualSync(LibrarySync):
def __init__(self):
LibrarySync.__init__(self)
def sync(self, mediatype=None):
if mediatype in ('movies', 'boxsets', 'musicvideos', 'tvshows'):
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn('video') as cursor_video:
pDialog = self.progressDialog("Manual Sync: %s" % mediatype)
if mediatype == 'movies':
self.movies(cursor_emby, cursor_video, pDialog)
elif mediatype == "boxsets":
self.boxsets(cursor_emby, cursor_video, pDialog)
elif mediatype =='musicvideos':
self.musicvideos(cursor_emby, cursor_video, pDialog)
elif mediatype == 'tvshows':
self.tvshows(cursor_emby, cursor_video, pDialog)
pDialog.close()
return
elif mediatype == 'music':
with database.DatabaseConn('emby') as cursor_emby:
with database.DatabaseConn('music') as cursor_music:
pDialog = self.progressDialog("Manual Sync: %s" % mediatype)
self.music(cursor_emby, cursor_music, pDialog)
pDialog.close()
return
else:
return self.fullSync(manualrun=True)
def movies(self, embycursor, kodicursor, pdialog):
return Movies(embycursor, kodicursor, pdialog).compare_all()
def boxsets(self, embycursor, kodicursor, pdialog):
return Movies(embycursor, kodicursor, pdialog).force_refresh_boxsets()
def musicvideos(self, embycursor, kodicursor, pdialog):
return MusicVideos(embycursor, kodicursor, pdialog).compare_all()
def tvshows(self, embycursor, kodicursor, pdialog):
return TVShows(embycursor, kodicursor, pdialog).compare_all()
def music(self, embycursor, kodicursor, pdialog):
return Music(embycursor, kodicursor).compare_all()