From 921da22c7d51c0ba2ca9c6964a83584953cf2307 Mon Sep 17 00:00:00 2001 From: mtlott Date: Sun, 13 Sep 2015 19:30:58 -0400 Subject: [PATCH] Updated Anime Plugin to work with current version of Emby - Fixed Null Reference to series.People causing crashes - Added support for MissingEpisodes - Added support for GetSearchResults during identify operation --- .gitattributes | 17 + .../MediaBrowser.Plugins.Anime.Tests.csproj | 34 +- .../packages.config | 11 +- .../MediaBrowser.Plugins.Anime.csproj | 30 +- .../Providers/AniDB/AniDbSeasonProvider.cs | 5 +- .../Providers/AniDB/AniDbSeriesProvider.cs | 129 ++++- .../AniList/AniListSeriesProvider.cs | 5 +- .../Providers/DummySeasonProvider.cs | 182 ++++++ .../Providers/MissingEpisodeProvider.cs | 527 ++++++++++++++++++ .../Providers/SeriesPostScanTask.cs | 143 +++++ MediaBrowser.Plugins.Anime/packages.config | 7 +- 11 files changed, 1046 insertions(+), 44 deletions(-) create mode 100644 .gitattributes create mode 100644 MediaBrowser.Plugins.Anime/Providers/DummySeasonProvider.cs create mode 100644 MediaBrowser.Plugins.Anime/Providers/MissingEpisodeProvider.cs create mode 100644 MediaBrowser.Plugins.Anime/Providers/SeriesPostScanTask.cs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bdb0cab --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/MediaBrowser.Plugins.Anime.Tests/MediaBrowser.Plugins.Anime.Tests.csproj b/MediaBrowser.Plugins.Anime.Tests/MediaBrowser.Plugins.Anime.Tests.csproj index b3f5acd..a323353 100644 --- a/MediaBrowser.Plugins.Anime.Tests/MediaBrowser.Plugins.Anime.Tests.csproj +++ b/MediaBrowser.Plugins.Anime.Tests/MediaBrowser.Plugins.Anime.Tests.csproj @@ -32,27 +32,32 @@ 4 - - False - ..\packages\MediaBrowser.Common.3.0.400\lib\net45\MediaBrowser.Common.dll + + ..\packages\Interfaces.IO.1.0.0.5\lib\portable-net45+sl4+wp71+win8+wpa81\Interfaces.IO.dll + True + + + ..\packages\MediaBrowser.Common.3.0.633\lib\net45\MediaBrowser.Common.dll + True .\MediaBrowser.Common.Implementations.dll - - False - ..\packages\MediaBrowser.Server.Core.3.0.400\lib\net45\MediaBrowser.Controller.dll + + ..\packages\MediaBrowser.Server.Core.3.0.633\lib\net45\MediaBrowser.Controller.dll + True - - False - ..\packages\MediaBrowser.Common.3.0.400\lib\net45\MediaBrowser.Model.dll + + ..\packages\MediaBrowser.Common.3.0.633\lib\net45\MediaBrowser.Model.dll + True - - False - ..\packages\Moq.4.2.1402.2112\lib\net40\Moq.dll + + ..\packages\Moq.4.2.1507.0118\lib\net40\Moq.dll + True - - ..\packages\NUnit.2.6.3\lib\nunit.framework.dll + + ..\packages\NUnit.2.6.4\lib\nunit.framework.dll + True @@ -81,6 +86,7 @@ PreserveNewest + Designer PreserveNewest diff --git a/MediaBrowser.Plugins.Anime.Tests/packages.config b/MediaBrowser.Plugins.Anime.Tests/packages.config index 8492340..1e8833d 100644 --- a/MediaBrowser.Plugins.Anime.Tests/packages.config +++ b/MediaBrowser.Plugins.Anime.Tests/packages.config @@ -1,9 +1,8 @@  - - - - - - + + + + + \ No newline at end of file diff --git a/MediaBrowser.Plugins.Anime/MediaBrowser.Plugins.Anime.csproj b/MediaBrowser.Plugins.Anime/MediaBrowser.Plugins.Anime.csproj index 4c05051..ac7ecda 100644 --- a/MediaBrowser.Plugins.Anime/MediaBrowser.Plugins.Anime.csproj +++ b/MediaBrowser.Plugins.Anime/MediaBrowser.Plugins.Anime.csproj @@ -32,20 +32,25 @@ 4 - - False - ..\packages\MediaBrowser.Common.3.0.400\lib\net45\MediaBrowser.Common.dll + + ..\packages\Interfaces.IO.1.0.0.5\lib\portable-net45+sl4+wp71+win8+wpa81\Interfaces.IO.dll + True - - False - ..\packages\MediaBrowser.Server.Core.3.0.400\lib\net45\MediaBrowser.Controller.dll + + ..\packages\MediaBrowser.Common.3.0.633\lib\net45\MediaBrowser.Common.dll + True - - False - ..\packages\MediaBrowser.Common.3.0.400\lib\net45\MediaBrowser.Model.dll + + ..\packages\MediaBrowser.Server.Core.3.0.633\lib\net45\MediaBrowser.Controller.dll + True - - ..\packages\morelinq.1.0.16006\lib\net35\MoreLinq.dll + + ..\packages\MediaBrowser.Common.3.0.633\lib\net45\MediaBrowser.Model.dll + True + + + ..\packages\morelinq.1.1.1\lib\net35\MoreLinq.dll + True @@ -77,10 +82,13 @@ + + + diff --git a/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeasonProvider.cs b/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeasonProvider.cs index 7bebc1e..5b7b5fc 100644 --- a/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeasonProvider.cs +++ b/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeasonProvider.cs @@ -7,6 +7,7 @@ using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Providers; @@ -18,10 +19,10 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB private readonly SeriesIndexSearch _indexSearcher; private readonly AniDbSeriesProvider _seriesProvider; - public AniDbSeasonProvider(IServerConfigurationManager configurationManager, IHttpClient httpClient, IApplicationPaths appPaths) + public AniDbSeasonProvider(IServerConfigurationManager configurationManager, IHttpClient httpClient, IApplicationPaths appPaths, ILibraryManager library) { _indexSearcher = new SeriesIndexSearch(configurationManager, httpClient); - _seriesProvider = new AniDbSeriesProvider(appPaths, httpClient, configurationManager); + _seriesProvider = new AniDbSeriesProvider(appPaths, httpClient, configurationManager, library); } public async Task> GetMetadata(SeasonInfo info, CancellationToken cancellationToken) diff --git a/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeriesProvider.cs b/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeriesProvider.cs index 196eac6..a0b2bf0 100644 --- a/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeriesProvider.cs +++ b/MediaBrowser.Plugins.Anime/Providers/AniDB/AniDbSeriesProvider.cs @@ -17,6 +17,7 @@ using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; @@ -31,6 +32,10 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB private const string SeriesQueryUrl = "http://api.anidb.net:9001/httpapi?request=anime&client={0}&clientver=1&protover=1&aid={1}"; private const string ClientName = "mediabrowser"; + private const string TvdbSeriesOffset = "TvdbSeriesOffset"; + private const string TvdbSeriesOffsetFormat = "{0}-{1}"; + internal static AniDbSeriesProvider Current { get; private set; } + // AniDB has very low request rate limits, a minimum of 2 seconds between requests, and an average of 4 seconds between requests public static readonly SemaphoreSlim ResourcePool = new SemaphoreSlim(1, 1); public static readonly RateLimiter RequestLimiter = new RateLimiter(TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(5)); @@ -39,6 +44,7 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB private static readonly Regex AniDbUrlRegex = new Regex(@"http://anidb.net/\w+ \[(?[^\]]*)\]"); private readonly IApplicationPaths _appPaths; private readonly IHttpClient _httpClient; + private readonly ILibraryManager _libraryManager; private readonly SeriesIndexSearch _indexSearcher; private readonly Dictionary _typeMappings = new Dictionary @@ -48,13 +54,16 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB {"Chief Animation Direction", "Chief Animation Director"} }; - public AniDbSeriesProvider(IApplicationPaths appPaths, IHttpClient httpClient, IServerConfigurationManager configurationManager) + public AniDbSeriesProvider(IApplicationPaths appPaths, IHttpClient httpClient, IServerConfigurationManager configurationManager, ILibraryManager libraryManager) { _appPaths = appPaths; _httpClient = httpClient; _indexSearcher = new SeriesIndexSearch(configurationManager, httpClient); + _libraryManager = libraryManager; TitleMatcher = AniDbTitleMatcher.DefaultInstance; + + Current = this; } public IAniDbTitleMatcher TitleMatcher { get; set; } @@ -87,9 +96,30 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB get { return "AniDB"; } } - public Task> GetSearchResults(SeriesInfo searchInfo, CancellationToken cancellationToken) + public async Task> GetSearchResults(SeriesInfo searchInfo, CancellationToken cancellationToken) { - return Task.FromResult(Enumerable.Empty()); + var seriesId = searchInfo.GetProviderId(ProviderNames.AniDb); + + var metadata = await GetMetadata(searchInfo, cancellationToken).ConfigureAwait(false); + + var list = new List(); + + if (metadata.HasMetadata) + { + var res = new RemoteSearchResult + { + Name = metadata.Item.Name, + PremiereDate = metadata.Item.PremiereDate, + ProductionYear = metadata.Item.ProductionYear, + ProviderIds = metadata.Item.ProviderIds, + SearchProviderName = Name + }; + + list.Add(res); + } + + return list; + //return Task.FromResult(Enumerable.Empty()); } public Task GetImageResponse(string url, CancellationToken cancellationToken) @@ -221,6 +251,13 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB ParseCategories(series, subtree); } + break; + case "episodes": + using (XmlReader subtree = reader.ReadSubtree()) + { + ParseEpisodes(series, subtree); + } + break; } } @@ -231,6 +268,44 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB GenreHelper.RemoveDuplicateTags(series); } + private void ParseEpisodes(Series series, XmlReader reader) + { + var episodes = new List(); + + while (reader.Read()) + { + if (reader.NodeType == XmlNodeType.Element && reader.Name == "episode") + { + int id; + if (int.TryParse(reader.GetAttribute("id"), out id) && IgnoredCategoryIds.Contains(id)) + continue; + + using (XmlReader episodeSubtree = reader.ReadSubtree()) + { + while (episodeSubtree.Read()) + { + if (episodeSubtree.NodeType == XmlNodeType.Element) + { + switch (episodeSubtree.Name) + { + case "epno": + string epno = episodeSubtree.ReadElementContentAsString(); + //EpisodeInfo info = new EpisodeInfo(); + //info.AnimeSeriesIndex = series.AnimeSeriesIndex; + //info.IndexNumberEnd = string(epno); + //info.SeriesProviderIds.GetOrDefault(ProviderNames.AniDb); + //episodes.Add(info); + break; + } + } + } + } + } + } + + //series.Genres = genres.OrderBy(g => g.Weight).Select(g => g.Name).ToList(); + } + private void ParseCategories(Series series, XmlReader reader) { var genres = new List(); @@ -366,9 +441,9 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB } } - if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(role) && series.People.All(p => p.Name != name)) + if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(role)) // && series.People.All(p => p.Name != name)) { - series.People.Add(CreatePerson(name, PersonType.Actor, role)); + series.AddPerson(CreatePerson(name, PersonType.Actor, role)); } } @@ -437,7 +512,7 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB } else { - series.People.Add(CreatePerson(name, type)); + series.AddPerson(CreatePerson(name, type)); } } } @@ -741,6 +816,48 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniDB } public int Order { get { return -1; } } + + /// + /// Gets the series data path. + /// + /// The app paths. + /// The series id. + /// System.String. + internal static string GetSeriesDataPath(IApplicationPaths appPaths, string seriesId) + { + var seriesDataPath = Path.Combine(GetSeriesDataPath(appPaths), seriesId); + + return seriesDataPath; + } + + /// + /// Gets the series data path. + /// + /// The app paths. + /// System.String. + internal static string GetSeriesDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.CachePath, "anidb\\series"); + + return dataPath; + } + + internal static int? GetSeriesOffset(Dictionary seriesProviderIds) + { + string idString; + if (!seriesProviderIds.TryGetValue(TvdbSeriesOffset, out idString)) + return null; + + var parts = idString.Split('-'); + if (parts.Length < 2) + return null; + + int offset; + if (int.TryParse(parts[1], out offset)) + return offset; + + return null; + } } public class Title diff --git a/MediaBrowser.Plugins.Anime/Providers/AniList/AniListSeriesProvider.cs b/MediaBrowser.Plugins.Anime/Providers/AniList/AniListSeriesProvider.cs index 4102e7e..bc1d966 100644 --- a/MediaBrowser.Plugins.Anime/Providers/AniList/AniListSeriesProvider.cs +++ b/MediaBrowser.Plugins.Anime/Providers/AniList/AniListSeriesProvider.cs @@ -13,6 +13,7 @@ using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Providers; @@ -104,11 +105,11 @@ namespace MediaBrowser.Plugins.Anime.Providers.AniList private readonly ILogger _logger; private readonly AniDbSeriesProvider _anidbSeriesProvider; - public AniListSeriesProvider(ILogManager logManager, IApplicationPaths appPaths, IHttpClient httpClient, IServerConfigurationManager configurationManger) + public AniListSeriesProvider(ILogManager logManager, IApplicationPaths appPaths, IHttpClient httpClient, IServerConfigurationManager configurationManger, ILibraryManager library) { _downloader = new AniListSeriesDownloader(appPaths, logManager.GetLogger("AniList")); _logger = logManager.GetLogger("AniList"); - _anidbSeriesProvider = new AniDbSeriesProvider(appPaths, httpClient, configurationManger); + _anidbSeriesProvider = new AniDbSeriesProvider(appPaths, httpClient, configurationManger, library); } public async Task> GetMetadata(SeriesInfo info, CancellationToken cancellationToken) diff --git a/MediaBrowser.Plugins.Anime/Providers/DummySeasonProvider.cs b/MediaBrowser.Plugins.Anime/Providers/DummySeasonProvider.cs new file mode 100644 index 0000000..4de9f40 --- /dev/null +++ b/MediaBrowser.Plugins.Anime/Providers/DummySeasonProvider.cs @@ -0,0 +1,182 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Localization; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Plugins.Anime.Providers +{ + public class DummySeasonProvider + { + private readonly IServerConfigurationManager _config; + private readonly ILogger _logger; + private readonly ILocalizationManager _localization; + private readonly ILibraryManager _libraryManager; + + private readonly CultureInfo _usCulture = new CultureInfo("en-US"); + + public DummySeasonProvider(IServerConfigurationManager config, ILogger logger, ILocalizationManager localization, ILibraryManager libraryManager) + { + _config = config; + _logger = logger; + _localization = localization; + _libraryManager = libraryManager; + } + + public async Task Run(Series series, CancellationToken cancellationToken) + { + await RemoveObsoleteSeasons(series).ConfigureAwait(false); + + var hasNewSeasons = await AddDummySeasonFolders(series, cancellationToken).ConfigureAwait(false); + + if (hasNewSeasons) + { + var directoryService = new DirectoryService(_logger); + + //await series.RefreshMetadata(new MetadataRefreshOptions(directoryService), cancellationToken).ConfigureAwait(false); + + //await series.ValidateChildren(new Progress(), cancellationToken, new MetadataRefreshOptions(directoryService)) + // .ConfigureAwait(false); + } + } + + private async Task AddDummySeasonFolders(Series series, CancellationToken cancellationToken) + { + var episodesInSeriesFolder = series.GetRecursiveChildren() + .OfType() + .Where(i => !i.IsInSeasonFolder) + .ToList(); + + var hasChanges = false; + + // Loop through the unique season numbers + foreach (var seasonNumber in episodesInSeriesFolder.Select(i => i.ParentIndexNumber ?? -1) + .Where(i => i >= 0) + .Distinct() + .ToList()) + { + var hasSeason = series.Children.OfType() + .Any(i => i.IndexNumber.HasValue && i.IndexNumber.Value == seasonNumber); + + if (!hasSeason) + { + await AddSeason(series, seasonNumber, cancellationToken).ConfigureAwait(false); + + hasChanges = true; + } + } + + // Unknown season - create a dummy season to put these under + if (episodesInSeriesFolder.Any(i => !i.ParentIndexNumber.HasValue)) + { + var hasSeason = series.Children.OfType() + .Any(i => !i.IndexNumber.HasValue); + + if (!hasSeason) + { + await AddSeason(series, null, cancellationToken).ConfigureAwait(false); + + hasChanges = true; + } + } + + return hasChanges; + } + + /// + /// Adds the season. + /// + /// The series. + /// The season number. + /// The cancellation token. + /// Task{Season}. + public async Task AddSeason(Series series, + int? seasonNumber, + CancellationToken cancellationToken) + { + var seasonName = seasonNumber == 0 ? + _config.Configuration.SeasonZeroDisplayName : + (seasonNumber.HasValue ? string.Format(_localization.GetLocalizedString("NameSeasonNumber"), seasonNumber.Value.ToString(_usCulture)) : _localization.GetLocalizedString("NameSeasonUnknown")); + + _logger.Info("Creating Season {0} entry for {1}", seasonName, series.Name); + + var season = new Season + { + Name = seasonName, + IndexNumber = seasonNumber, + Id = (series.Id + (seasonNumber ?? -1).ToString(_usCulture) + seasonName).GetMBId(typeof(Season)) + }; + + season.SetParent(series); + + await series.AddChild(season, cancellationToken).ConfigureAwait(false); + + await season.RefreshMetadata(new MetadataRefreshOptions(), cancellationToken).ConfigureAwait(false); + + return season; + } + + private async Task RemoveObsoleteSeasons(Series series) + { + var existingSeasons = series.Children.OfType().ToList(); + + var physicalSeasons = existingSeasons + .Where(i => i.LocationType != LocationType.Virtual) + .ToList(); + + var virtualSeasons = existingSeasons + .Where(i => i.LocationType == LocationType.Virtual) + .ToList(); + + var episodes = series.GetRecursiveChildren().OfType().ToList(); + + var seasonsToRemove = virtualSeasons + .Where(i => + { + if (i.IndexNumber.HasValue) + { + var seasonNumber = i.IndexNumber.Value; + + // If there's a physical season with the same number, delete it + if (physicalSeasons.Any(p => p.IndexNumber.HasValue && (p.IndexNumber.Value == seasonNumber))) + { + return true; + } + + // If there are no episodes with this season number, delete it + if (episodes.All(e => !e.ParentIndexNumber.HasValue || e.ParentIndexNumber.Value != seasonNumber)) + { + return true; + } + + return false; + } + + // Season does not have a number + // Remove if there are no episodes directly in series without a season number + return episodes.All(s => s.ParentIndexNumber.HasValue || !s.IsInSeasonFolder); + }) + .ToList(); + + var hasChanges = false; + + foreach (var seasonToRemove in seasonsToRemove) + { + _logger.Info("Removing virtual season {0} {1}", seasonToRemove.Series.Name, seasonToRemove.IndexNumber); + + await _libraryManager.DeleteItem(seasonToRemove).ConfigureAwait(false); + + hasChanges = true; + } + + return hasChanges; + } + } +} diff --git a/MediaBrowser.Plugins.Anime/Providers/MissingEpisodeProvider.cs b/MediaBrowser.Plugins.Anime/Providers/MissingEpisodeProvider.cs new file mode 100644 index 0000000..cd0e6f8 --- /dev/null +++ b/MediaBrowser.Plugins.Anime/Providers/MissingEpisodeProvider.cs @@ -0,0 +1,527 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Localization; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; + +using MediaBrowser.Plugins.Anime.Providers.AniDB; + +namespace MediaBrowser.Plugins.Anime.Providers +{ + class MissingEpisodeProvider + { + private readonly IServerConfigurationManager _config; + private readonly ILogger _logger; + private readonly ILibraryManager _libraryManager; + private readonly ILocalizationManager _localization; + + private readonly CultureInfo _usCulture = new CultureInfo("en-US"); + + public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization) + { + _logger = logger; + _config = config; + _libraryManager = libraryManager; + _localization = localization; + } + + public async Task Run(IEnumerable> series, CancellationToken cancellationToken) + { + foreach (var seriesGroup in series) + { + try + { + await Run(seriesGroup, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (DirectoryNotFoundException) + { + _logger.Warn("Series files missing for series id {0}", seriesGroup.Key); + } + catch (Exception ex) + { + _logger.ErrorException("Error in missing episode provider for series id {0}", ex, seriesGroup.Key); + } + } + } + + private async Task Run(IGrouping group, CancellationToken cancellationToken) + { + var tvdbId = group.Key; + + var seriesDataPath = AniDbSeriesProvider.GetSeriesDataPath(_config.ApplicationPaths, tvdbId); + + var episodeFiles = Directory.EnumerateFiles(seriesDataPath, "*.xml", SearchOption.TopDirectoryOnly) + .Select(Path.GetFileNameWithoutExtension) + .Where(i => i.StartsWith("episode-", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + var episodeLookup = episodeFiles + .Select(i => + { + var parts = i.Split('-'); + + if (parts.Length == 2) + { + int seasonNumber = 1; + + // if (int.TryParse(parts[1], NumberStyles.Integer, _usCulture, out seasonNumber)) + // { + int episodeNumber; + + //if (int.TryParse(parts[2], NumberStyles.Integer, _usCulture, out episodeNumber)) + if (int.TryParse(parts[1], NumberStyles.Integer, _usCulture, out episodeNumber)) + { + return new Tuple(seasonNumber, episodeNumber); + } + // } + } + + return new Tuple(-1, -1); + }) + .Where(i => i.Item1 != -1 && i.Item2 != -1) + .ToList(); + + var hasBadData = HasInvalidContent(group); + + var anySeasonsRemoved = await RemoveObsoleteOrMissingSeasons(group, episodeLookup) + .ConfigureAwait(false); + + var anyEpisodesRemoved = await RemoveObsoleteOrMissingEpisodes(group, episodeLookup) + .ConfigureAwait(false); + + var hasNewEpisodes = false; + + if (_config.Configuration.EnableInternetProviders) + { + var seriesConfig = _config.Configuration.MetadataOptions.FirstOrDefault(i => string.Equals(i.ItemType, typeof(Series).Name, StringComparison.OrdinalIgnoreCase)); + + if (seriesConfig == null || !seriesConfig.DisabledMetadataFetchers.Contains(AniDbSeriesProvider.Current.Name, StringComparer.OrdinalIgnoreCase)) + { + hasNewEpisodes = await AddMissingEpisodes(group.ToList(), hasBadData, seriesDataPath, episodeLookup, cancellationToken) + .ConfigureAwait(false); + } + } + + if (hasNewEpisodes || anySeasonsRemoved || anyEpisodesRemoved) + { + foreach (var series in group) + { + var directoryService = new DirectoryService(_logger); + + await series.RefreshMetadata(new MetadataRefreshOptions() + { + }, cancellationToken).ConfigureAwait(false); + + await series.ValidateChildren(new Progress(), cancellationToken, new MetadataRefreshOptions(), true) + .ConfigureAwait(false); + } + } + } + + /// + /// Returns true if a series has any seasons or episodes without season or episode numbers + /// If this data is missing no virtual items will be added in order to prevent possible duplicates + /// + /// + /// + private bool HasInvalidContent(IEnumerable group) + { + var allItems = group.ToList().SelectMany(i => i.GetRecursiveChildren()).ToList(); + + return allItems.OfType().Any(i => !i.IndexNumber.HasValue) || + allItems.OfType().Any(i => + { + if (!i.ParentIndexNumber.HasValue) + { + return true; + } + + // You could have episodes under season 0 with no number + return false; + }); + } + + /// + /// Adds the missing episodes. + /// + /// The series. + /// if set to true [series has bad data]. + /// The series data path. + /// The episode lookup. + /// The cancellation token. + /// Task. + private async Task AddMissingEpisodes(List series, + bool seriesHasBadData, + string seriesDataPath, + IEnumerable> episodeLookup, + CancellationToken cancellationToken) + { + var existingEpisodes = (from s in series + let seasonOffset = AniDbSeriesProvider.GetSeriesOffset(s.ProviderIds) ?? ((s.AnimeSeriesIndex ?? 1) - 1) + from c in s.GetRecursiveChildren().OfType() + select new Tuple((c.ParentIndexNumber ?? 0) + seasonOffset, c)) + .ToList(); + + var lookup = episodeLookup as IList> ?? episodeLookup.ToList(); + + var seasonCounts = (from e in lookup + group e by e.Item1 into g + select g) + .ToDictionary(g => g.Key, g => g.Count()); + + var hasChanges = false; + + foreach (var tuple in lookup) + { + if (tuple.Item1 <= 0) + { + // Ignore season zeros + continue; + } + + if (tuple.Item2 <= 0) + { + // Ignore episode zeros + continue; + } + + var existingEpisode = GetExistingEpisode(existingEpisodes, seasonCounts, tuple); + + if (existingEpisode != null) + { + continue; + } + + var airDate = GetAirDate(seriesDataPath, tuple.Item1, tuple.Item2); + + if (!airDate.HasValue) + { + continue; + } + var now = DateTime.UtcNow; + + var targetSeries = DetermineAppropriateSeries(series, tuple.Item1); + var seasonOffset = AniDbSeriesProvider.GetSeriesOffset(targetSeries.ProviderIds) ?? ((targetSeries.AnimeSeriesIndex ?? 1) - 1); + + if (airDate.Value < now) + { + // Be conservative here to avoid creating missing episodes for ones they already have + if (!seriesHasBadData) + { + // tvdb has a lot of nearly blank episodes + _logger.Info("Creating virtual missing episode {0} {1}x{2}", targetSeries.Name, tuple.Item1, tuple.Item2); + await AddEpisode(targetSeries, tuple.Item1 - seasonOffset, tuple.Item2, cancellationToken).ConfigureAwait(false); + + hasChanges = true; + } + } + else if (airDate.Value > now) + { + // tvdb has a lot of nearly blank episodes + _logger.Info("Creating virtual unaired episode {0} {1}x{2}", targetSeries.Name, tuple.Item1, tuple.Item2); + await AddEpisode(targetSeries, tuple.Item1 - seasonOffset, tuple.Item2, cancellationToken).ConfigureAwait(false); + + hasChanges = true; + } + } + + return hasChanges; + } + + private Series DetermineAppropriateSeries(IEnumerable series, int seasonNumber) + { + var seriesAndOffsets = series.Select(s => new { Series = s, SeasonOffset = AniDbSeriesProvider.GetSeriesOffset(s.ProviderIds) ?? ((s.AnimeSeriesIndex ?? 1) - 1) }).ToList(); + + var bestMatch = seriesAndOffsets.FirstOrDefault(s => s.Series.GetRecursiveChildren().OfType().Any(season => (season.IndexNumber + s.SeasonOffset) == seasonNumber)) ?? + seriesAndOffsets.FirstOrDefault(s => s.Series.GetRecursiveChildren().OfType().Any(season => (season.IndexNumber + s.SeasonOffset) == 1)) ?? + seriesAndOffsets.OrderBy(s => s.Series.GetRecursiveChildren().OfType().Select(season => season.IndexNumber + s.SeasonOffset).Min()).First(); + + return bestMatch.Series; + } + + /// + /// Removes the virtual entry after a corresponding physical version has been added + /// + private async Task RemoveObsoleteOrMissingEpisodes(IEnumerable series, + IEnumerable> episodeLookup) + { + var existingEpisodes = (from s in series + let seasonOffset = AniDbSeriesProvider.GetSeriesOffset(s.ProviderIds) ?? ((s.AnimeSeriesIndex ?? 1) - 1) + from c in s.GetRecursiveChildren().OfType() + select new { SeasonOffset = seasonOffset, Episode = c }) + .ToList(); + + var physicalEpisodes = existingEpisodes + .Where(i => i.Episode.LocationType != LocationType.Virtual) + .ToList(); + + var virtualEpisodes = existingEpisodes + .Where(i => i.Episode.LocationType == LocationType.Virtual) + .ToList(); + + var episodesToRemove = virtualEpisodes + .Where(i => + { + if (i.Episode.IndexNumber.HasValue && i.Episode.ParentIndexNumber.HasValue) + { + var seasonNumber = i.Episode.ParentIndexNumber.Value + i.SeasonOffset; + var episodeNumber = i.Episode.IndexNumber.Value; + + // If there's a physical episode with the same season and episode number, delete it + if (physicalEpisodes.Any(p => + p.Episode.ParentIndexNumber.HasValue && (p.Episode.ParentIndexNumber.Value + p.SeasonOffset) == seasonNumber && + p.Episode.ContainsEpisodeNumber(episodeNumber))) + { + return true; + } + + // If the episode no longer exists in the remote lookup, delete it + if (!episodeLookup.Any(e => e.Item1 == seasonNumber && e.Item2 == episodeNumber)) + { + return true; + } + + return false; + } + + return true; + }) + .ToList(); + + var hasChanges = false; + + foreach (var episodeToRemove in episodesToRemove.Select(e => e.Episode)) + { + _logger.Info("Removing missing/unaired episode {0} {1}x{2}", episodeToRemove.Series.Name, episodeToRemove.ParentIndexNumber, episodeToRemove.IndexNumber); + + await _libraryManager.DeleteItem(episodeToRemove).ConfigureAwait(false); + + hasChanges = true; + } + + return hasChanges; + } + + /// + /// Removes the obsolete or missing seasons. + /// + /// The series. + /// The episode lookup. + /// Task{System.Boolean}. + private async Task RemoveObsoleteOrMissingSeasons(IEnumerable series, + IEnumerable> episodeLookup) + { + var existingSeasons = (from s in series + let seasonOffset = AniDbSeriesProvider.GetSeriesOffset(s.ProviderIds) ?? ((s.AnimeSeriesIndex ?? 1) - 1) + from c in s.Children.OfType() + select new { SeasonOffset = seasonOffset, Season = c }) + .ToList(); + + var physicalSeasons = existingSeasons + .Where(i => i.Season.LocationType != LocationType.Virtual) + .ToList(); + + var virtualSeasons = existingSeasons + .Where(i => i.Season.LocationType == LocationType.Virtual) + .ToList(); + + var seasonsToRemove = virtualSeasons + .Where(i => + { + if (i.Season.IndexNumber.HasValue) + { + var seasonNumber = i.Season.IndexNumber.Value + i.SeasonOffset; + + // If there's a physical season with the same number, delete it + if (physicalSeasons.Any(p => p.Season.IndexNumber.HasValue && (p.Season.IndexNumber.Value + p.SeasonOffset) == seasonNumber)) + { + return true; + } + + // If the season no longer exists in the remote lookup, delete it + if (episodeLookup.All(e => e.Item1 != seasonNumber)) + { + return true; + } + + return false; + } + + // Season does not have a number + // Remove if there are no episodes directly in series without a season number + return i.Season.Series.GetRecursiveChildren().OfType().All(s => s.ParentIndexNumber.HasValue || !s.IsInSeasonFolder); + }) + .ToList(); + + var hasChanges = false; + + foreach (var seasonToRemove in seasonsToRemove.Select(s => s.Season)) + { + _logger.Info("Removing virtual season {0} {1}", seasonToRemove.Series.Name, seasonToRemove.IndexNumber); + + await _libraryManager.DeleteItem(seasonToRemove).ConfigureAwait(false); + + hasChanges = true; + } + + return hasChanges; + } + + /// + /// Adds the episode. + /// + /// The series. + /// The season number. + /// The episode number. + /// The cancellation token. + /// Task. + private async Task AddEpisode(Series series, int seasonNumber, int episodeNumber, CancellationToken cancellationToken) + { + var season = series.Children.OfType() + .FirstOrDefault(i => i.IndexNumber.HasValue && i.IndexNumber.Value == seasonNumber); + + if (season == null) + { + var provider = new DummySeasonProvider(_config, _logger, _localization, _libraryManager); + season = await provider.AddSeason(series, seasonNumber, cancellationToken).ConfigureAwait(false); + } + + var name = string.Format("Episode {0}", episodeNumber.ToString(_usCulture)); + + var episode = new Episode + { + Name = name, + IndexNumber = episodeNumber, + ParentIndexNumber = seasonNumber, + Id = (series.Id + seasonNumber.ToString(_usCulture) + name).GetMBId(typeof(Episode)) + }; + + episode.SetParent(season); + + await season.AddChild(episode, cancellationToken).ConfigureAwait(false); + + await episode.RefreshMetadata(new MetadataRefreshOptions + { + }, cancellationToken).ConfigureAwait(false); + } + + /// + /// Gets the existing episode. + /// + /// The existing episodes. + /// + /// The tuple. + /// Episode. + private Episode GetExistingEpisode(IList> existingEpisodes, Dictionary seasonCounts, Tuple tuple) + { + var s = tuple.Item1; + var e = tuple.Item2; + + while (true) + { + var episode = GetExistingEpisode(existingEpisodes, s, e); + if (episode != null) + return episode; + + s--; + + if (seasonCounts.ContainsKey(s)) + e += seasonCounts[s]; + else + break; + } + + return null; + } + + private static Episode GetExistingEpisode(IEnumerable> existingEpisodes, int season, int episode) + { + return existingEpisodes + .Where(i => i.Item1 == season && i.Item2.ContainsEpisodeNumber(episode)) + .Select(i => i.Item2) + .FirstOrDefault(); + } + + /// + /// Gets the air date. + /// + /// The series data path. + /// The season number. + /// The episode number. + /// System.Nullable{DateTime}. + private DateTime? GetAirDate(string seriesDataPath, int seasonNumber, int episodeNumber) + { + // First open up the tvdb xml file and make sure it has valid data + //var filename = string.Format("episode-{0}-{1}.xml", seasonNumber.ToString(_usCulture), episodeNumber.ToString(_usCulture)); + var filename = string.Format("episode-{1}.xml", seasonNumber.ToString(_usCulture), episodeNumber.ToString(_usCulture)); + + var xmlPath = Path.Combine(seriesDataPath, filename); + + DateTime? airDate = null; + + // It appears the best way to filter out invalid entries is to only include those with valid air dates + using (var streamReader = new StreamReader(xmlPath, Encoding.UTF8)) + { + // Use XmlReader for best performance + using (var reader = XmlReader.Create(streamReader, new XmlReaderSettings + { + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true, + ValidationType = ValidationType.None + })) + { + reader.MoveToContent(); + + // Loop through each element + while (reader.Read()) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "airdate": + { + var val = reader.ReadElementContentAsString(); + + if (!string.IsNullOrWhiteSpace(val)) + { + DateTime date; + if (DateTime.TryParse(val, out date)) + { + airDate = date.ToUniversalTime(); + } + } + + break; + } + + default: + reader.Skip(); + break; + } + } + } + } + } + + return airDate; + } + } +} diff --git a/MediaBrowser.Plugins.Anime/Providers/SeriesPostScanTask.cs b/MediaBrowser.Plugins.Anime/Providers/SeriesPostScanTask.cs new file mode 100644 index 0000000..6780f4c --- /dev/null +++ b/MediaBrowser.Plugins.Anime/Providers/SeriesPostScanTask.cs @@ -0,0 +1,143 @@ +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Localization; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Plugins.Anime.Providers +{ + class SeriesGroup : List, IGrouping + { + public string Key { get; set; } + } + + class SeriesPostScanTask : ILibraryPostScanTask, IHasOrder + { + /// + /// The _library manager + /// + private readonly ILibraryManager _libraryManager; + private readonly IServerConfigurationManager _config; + private readonly ILogger _logger; + private readonly ILocalizationManager _localization; + + public SeriesPostScanTask(ILibraryManager libraryManager, ILogger logger, IServerConfigurationManager config, ILocalizationManager localization) + { + _libraryManager = libraryManager; + _logger = logger; + _config = config; + _localization = localization; + } + + public Task Run(IProgress progress, CancellationToken cancellationToken) + { + return RunInternal(progress, cancellationToken); + } + + private async Task RunInternal(IProgress progress, CancellationToken cancellationToken) + { + var seriesList = _libraryManager.RootFolder + .GetRecursiveChildren(i => i is Series) + .Cast() + .ToList(); + + var seriesGroups = FindSeriesGroups(seriesList).Where(g => !string.IsNullOrEmpty(g.Key)).ToList(); + + await new MissingEpisodeProvider(_logger, _config, _libraryManager, _localization).Run(seriesGroups, cancellationToken).ConfigureAwait(false); + + var numComplete = 0; + + foreach (var series in seriesList) + { + cancellationToken.ThrowIfCancellationRequested(); + + var episodes = series.GetRecursiveChildren(i => i is Episode) + .Cast() + .ToList(); + + var physicalEpisodes = episodes.Where(i => i.LocationType != LocationType.Virtual) + .ToList(); + + series.SeasonCount = episodes + .Select(i => i.ParentIndexNumber ?? 0) + .Where(i => i != 0) + .Distinct() + .Count(); + + series.SpecialFeatureIds = physicalEpisodes + .Where(i => i.ParentIndexNumber.HasValue && i.ParentIndexNumber.Value == 0) + .Select(i => i.Id) + .ToList(); + + numComplete++; + double percent = numComplete; + percent /= seriesList.Count; + percent *= 100; + + progress.Report(percent); + } + } + + private IEnumerable> FindSeriesGroups(List seriesList) + { + var links = seriesList.ToDictionary(s => s, s => seriesList.Where(c => c != s && ShareProviderId(s, c)).ToList()); + + var visited = new HashSet(); + + foreach (var series in seriesList) + { + if (!visited.Contains(series)) + { + var group = new SeriesGroup(); + FindAllLinked(series, visited, links, group); + + group.Key = group.Select(s => s.GetProviderId(ProviderNames.AniDb)).FirstOrDefault(id => !string.IsNullOrEmpty(id)); + + yield return group; + } + } + } + + private void FindAllLinked(Series series, HashSet visited, IDictionary> linksMap, List results) + { + results.Add(series); + visited.Add(series); + + var links = linksMap[series]; + + foreach (var s in links) + { + if (!visited.Contains(s)) + { + FindAllLinked(s, visited, linksMap, results); + } + } + } + + private bool ShareProviderId(Series a, Series b) + { + return a.ProviderIds.Any(id => + { + string value; + return b.ProviderIds.TryGetValue(id.Key, out value) && id.Value == value; + }); + } + + public int Order + { + get + { + // Run after tvdb update task + return 1; + } + } + } + +} diff --git a/MediaBrowser.Plugins.Anime/packages.config b/MediaBrowser.Plugins.Anime/packages.config index cc06bb5..768c931 100644 --- a/MediaBrowser.Plugins.Anime/packages.config +++ b/MediaBrowser.Plugins.Anime/packages.config @@ -1,6 +1,7 @@  - - - + + + + \ No newline at end of file