using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Data;
using Data.Enum;
using Data.Model;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Newtonsoft.Json;
namespace DanbooruDownloader.HostedServices
{
public class FindWaifuService : IHostedService
{
private const double MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION = 0.85d;
/// <summary>
/// Фильтр фажества. Минимальный процент
/// </summary>
private const double MIN_COUNT_PERCENT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING = 0.28;
private const double MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING = 10;
private const double MAX_FAG_POINTS_PER_CHARACTER = 10;
private const double FAG_POINTS_LOG10_MULTIPLIER = 0.15;
private const int MIN_CHAR_COUNT_TO_CREATE_STAT = 10;
private const int MAX_CHARACTERS_PER_IMAGE_FOR_FAGGING_COUNT = 2;
private readonly NFContextFactory NFContextFactory;
// ReSharper disable once NotAccessedField.Local
private readonly IConfiguration Configuration;
private readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
private CancellationToken CancellationToken => CancellationTokenSource.Token;
private Task MainTaskItself;
private Task Count123MeanTaskItself;
public FindWaifuService(
NFContextFactory nfContextFactory,
IConfiguration configuration
)
{
NFContextFactory = nfContextFactory;
Configuration = configuration;
}
public Task StartAsync(CancellationToken cancellationToken)
{
MainTaskItself = Task.Run(SearchForWaifu, cancellationToken);
Count123MeanTaskItself = Task.CompletedTask; // Task.Run(CalculateMeanCount123, cancellationToken);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken cancellationToken)
{
CancellationTokenSource.Cancel();
await MainTaskItself;
await Count123MeanTaskItself;
}
#region SearchForWaifu
private DanbooruTag[] tags;
private Dictionary<int, DanbooruTag> tagDictionary;
private Dictionary<int, int> countsByTagId;
private ICollection<int> tagArtistIds;
private ICollection<int> tagCharacterIds;
private ICollection<DanbooruTagIntersect> intersects;
private IDictionary<int, int> parentImages;
private int GetRealImageId(int imageId)
{
return parentImages.ContainsKey(imageId) ? parentImages[imageId] : imageId;
}
private IDictionary<int, int> characterToCopyright;
private int tagOriginalId;
private async Task SearchForWaifu_GenerateCopyright(NFContext context)
{
var tagCopyrightIds = tags
.Where(x => x.type == DanbooruTagType.Copyright)
.Where(x => x.id != tagOriginalId)
.Select(t => t.id)
.ToHashSet();
Console.WriteLine(
"{0}\ttagCopyrightIds.Length = {1,7}",
DateTimeOffset.UtcNow,
tagCopyrightIds.Count()
);
var i1 = await context
.DanbooruTagIntersect
.Where(x => tagCopyrightIds.Contains(x.tag_id))
.ToArrayAsync();
Console.WriteLine(
"{0}\tDb: i1.Length = {1,7}",
DateTimeOffset.UtcNow,
i1.Count()
);
i1 = i1
.Select(t => (art_id: GetRealImageId(t.art_id), t.tag_id))
.Distinct()
.Select(t => new DanbooruTagIntersect() {art_id = t.art_id, tag_id = t.tag_id})
.ToArray();
Console.WriteLine(
"{0}\tParented: i1.Length = {1,7}",
DateTimeOffset.UtcNow,
i1.Count()
);
var franchisesByArtId = i1
.GroupBy(t => t.art_id)
.Select(t => (art_id: t.Key, copyrights: t.Select(t1 => t1.tag_id).Distinct().ToArray()))
.Where(x => x.copyrights.Length == 1)
.ToDictionary(
t => t.art_id,
t => t.copyrights.First()
);
var charCounts1 = intersects
.Where(x => tagCharacterIds.Contains(x.tag_id))
.GroupBy(t => t.tag_id)
.ToDictionary(
t => t.Key,
t => t.Count()
);
var charCounts2 = intersects
.Where(x => franchisesByArtId.ContainsKey(x.art_id) && tagCharacterIds.Contains(x.tag_id))
.GroupBy(t => t.tag_id)
.ToDictionary(
t => t.Key,
t => t.Count()
);
Console.WriteLine(
"{0}\tfranchisesByArtId.Length = {1,7}" +
"\n\t\t\t\tcharCounts1.Length = {2,7}" +
"\n\t\t\t\tcharCounts2.Length = {3,7}",
DateTimeOffset.UtcNow,
franchisesByArtId.Count(),
charCounts1.Count(),
charCounts2.Count()
);
characterToCopyright = intersects
.Where(x => tagCharacterIds.Contains(x.tag_id) && charCounts1.ContainsKey(x.tag_id))
.Select(t =>
{
// ReSharper disable once ConvertToLambdaExpression
return (charId: t.tag_id,
copyright: franchisesByArtId.ContainsKey(t.art_id) ? franchisesByArtId[t.art_id] : -1);
})
.Where(x => x.copyright != -1)
.GroupBy(t => t.charId)
.Select(t =>
{
var charId = t.Key;
var minCount = charCounts1[charId] * MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION;
var (copyright, count) = t
.GroupBy(t1 => t1.copyright)
.Select(t1 => (t1.Key, count: t1.Count()))
.OrderByDescending(t1 => t1.count)
.First();
// ReSharper disable once ConvertIfStatementToReturnStatement
if (count >= minCount)
{
return (charId, copyright: copyright);
}
if (!charCounts2.ContainsKey(charId))
{
return (charId, copyright: -1);
}
minCount = charCounts2[charId] * MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION;
// ReSharper disable once ConvertIfStatementToReturnStatement
if (count >= minCount)
{
return (charId, copyright: copyright);
}
return (charId, copyright: -1);
})
.Where(x => x.copyright != -1)
.OrderBy(t => t.charId)
.ToDictionary(
t => t.charId,
t => t.copyright
);
Console.WriteLine(
"{0}\tcharacterToCopyright.Length = {1,7}",
DateTimeOffset.UtcNow,
characterToCopyright.Count()
);
}
private async Task SearchForWaifu_Step1()
{
Console.WriteLine(
"{0}\tSearchForWaifu_Step1",
DateTimeOffset.UtcNow
);
await using var context = NFContextFactory.CreateContext();
Console.WriteLine(
"{0}\tDB connected",
DateTimeOffset.UtcNow
);
parentImages = await context
.DanbooruImage
.Where(x => x.parent_id != null)
.ToDictionaryAsync(
t => t.id,
t => t.parent_id!.Value
);
Console.WriteLine(
"{0}\tparentImages.Length = {1,7}",
DateTimeOffset.UtcNow,
parentImages.Count()
);
tags = await context
.DanbooruTag
.ToArrayAsync();
tagDictionary = tags.ToDictionary(t => t.id, t => t);
Console.WriteLine(
"{0}\ttags.Length = {1,7}",
DateTimeOffset.UtcNow,
tags.Count()
);
tagOriginalId = tags.First(x => x.tag == "original").id;
tagArtistIds = tags
.Where(x => x.type == DanbooruTagType.Artist)
.Select(t => t.id)
.ToHashSet();
tagCharacterIds = tags
.Where(x => x.type == DanbooruTagType.Character)
.Select(t => t.id)
.ToHashSet();
var artistCharacterIds = tagArtistIds.Concat(tagCharacterIds).ToArray();
Console.WriteLine(
"{0}\ttagArtistIds.Length = {1,7}" +
"\n\t\t\t\ttagCharacterIds = {2,7}" +
"\n\t\t\t\tOriginal tag = {3,7}",
DateTimeOffset.UtcNow,
tagArtistIds.Count(),
tagCharacterIds.Count(),
tagOriginalId
);
var sw = Stopwatch.StartNew();
intersects = await context
.DanbooruTagIntersect
.Where(x => artistCharacterIds.Contains(x.tag_id))
.ToArrayAsync();
sw.Stop();
var intersectsRawCount = intersects.Count;
Console.WriteLine(
"{0}\tDb: intersects.Length = {1,7}",
DateTimeOffset.UtcNow,
intersects.Count()
);
intersects = intersects
.Select(t => (art_id: GetRealImageId(t.art_id), t.tag_id))
.Distinct()
.Select(t => new DanbooruTagIntersect()
{
art_id = t.art_id,
tag_id = t.tag_id,
})
.ToArray();
Console.WriteLine(
"{0}\tParented: intersects.Length = {1,7}",
DateTimeOffset.UtcNow,
intersects.Count()
);
await SearchForWaifu_GenerateCopyright(context);
{
var artId1 = intersects
.Where(x => tagArtistIds.Contains(x.tag_id))
.Select(t => t.art_id)
.ToHashSet();
artId1 = intersects
.Where(x => tagCharacterIds.Contains(x.tag_id))
.Select(t => t.art_id)
.Where(x => artId1.Contains(x))
.ToHashSet();
Console.WriteLine(
"{0}\tClear artId1 with characters and artists = {1,7}",
DateTimeOffset.UtcNow,
artId1.Count()
);
intersects = intersects
.Where(x => artId1.Contains(x.art_id))
.ToArray();
}
ClearIntersectsFromMultipleChars();
await ClearIntersectsFromBadTags(context);
countsByTagId = intersects
.GroupBy(t => t.tag_id)
.ToDictionary(
t => t.Key,
t => t.Count()
);
Console.WriteLine(
"{0}\tGet intersects: {6}" +
"\n\t\t\t\tArtists count = {1,6}" +
"\n\t\t\t\tCharacter count = {2,6}" +
"\n\t\t\t\tIntersects {3,7} ({4,7} {5:P1})" +
"\n\t\t\t\tcountsByTagId.Length = {7,7}",
DateTimeOffset.UtcNow,
tagArtistIds.Count,
tagCharacterIds.Count,
intersects.Count,
intersectsRawCount,
intersects.Count * 1d / intersectsRawCount,
sw.Elapsed,
countsByTagId.Count()
);
}
/// <summary>
/// Delete game_cg & Delete official art
/// </summary>
/// <param name="context"></param>
private async Task ClearIntersectsFromBadTags(NFContext context)
{
var gameCG = tags.First(x => x.tag == "game_cg").id;
var officialArt = tags.First(x => x.tag == "official_art").id;
var a = new[] {gameCG, officialArt};
ICollection<int> badArtIds = await context
.DanbooruTagIntersect
.Where(x => a.Contains(x.tag_id))
.Select(t => t.art_id)
.Distinct()
.ToArrayAsync();
badArtIds = badArtIds.ToHashSet();
intersects = intersects
.Where(x => !badArtIds.Contains(x.art_id))
.ToArray();
Console.WriteLine(
"{0}\tClearIntersectsFromBadTags: badArtIds.Length = {1,7}" +
"\n\t\t\t\tintersects.Length = {2,7}",
DateTimeOffset.UtcNow,
badArtIds.Count(),
intersects.Count()
);
}
/// <summary>
/// Delete multi-characters (4+) arts
/// </summary>
private void ClearIntersectsFromMultipleChars()
{
var badArtIds = intersects
.Where(x => tagCharacterIds.Contains(x.tag_id))
.GroupBy(t => t.art_id)
.Select(t => (art_id: t.Key, count: t.Count()))
.Where(x => x.count > MAX_CHARACTERS_PER_IMAGE_FOR_FAGGING_COUNT)
.Select(t => t.art_id)
.ToHashSet();
intersects = intersects
.Where(x => !badArtIds.Contains(x.art_id))
.ToArray();
Console.WriteLine(
"{0}\tClearIntersectsFromMultipleChars: badArtIds.Length = {1,7}" +
"\n\t\t\t\tintersects.Length = {2,7}",
DateTimeOffset.UtcNow,
badArtIds.Count(),
intersects.Count()
);
}
private async Task SearchForWaifu()
{
Console.WriteLine(
"{0}\tSearchForWaifu",
DateTimeOffset.UtcNow
);
await SearchForWaifu_Step1();
var byArts = intersects
.GroupBy(t => t.art_id)
.Select(t =>
{
var a = t.Select(t1 => t1.tag_id).ToArray();
var artists = a.Where(tagArtistIds.Contains).ToArray();
var characters = a.Where(tagCharacterIds.Contains).ToArray();
return (artId: t.Key, artists, characters);
})
.Where(x => x.artists.Any() && x.characters.Any())
.ToDictionary(
t => t.artId,
t => t
);
Console.WriteLine(
"{0}\tbyArts.Length = {1,7}",
DateTimeOffset.UtcNow,
byArts.Count()
);
var semiRawData = byArts
.Values
.SelectMany(t =>
{
var (_, artists, characters) = t;
return artists
.SelectMany(t1 => characters.Select(t2 => (artist: t1, character: t2)));
})
.GroupBy(t => t)
.Select(t => (key: t.Key, count: t.Count()))
.Select(t => (key: t.key, count: t.count, countP: t.count * 1d / countsByTagId[t.key.artist]))
.ToArray();
var countDictionary = semiRawData
.Where(x => x.count > MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING)
.OrderByDescending(t => t.countP + Math.Log10(t.count) * FAG_POINTS_LOG10_MULTIPLIER)
.ToDictionary(
t => t.key,
t => t.count
);
Console.WriteLine(
"{0}\tsemiRawData.Length = {1,7}" +
"\n\t\t\t\tcountDictionary = {2,7}",
DateTimeOffset.UtcNow,
semiRawData.Count(),
countDictionary.Count()
);
{
// ReSharper disable once NotAccessedVariable
var realCount = 0;
var lines = new List<string>();
foreach (var ((artist, character), count) in countDictionary /*.TakeWhile(_ => realCount < 5000)*/)
{
var artistCount = countsByTagId[artist];
var characterCount = countsByTagId[character];
if (characterCount - count < MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING)
{
continue;
}
if (count < artistCount * 0.1)
{
continue;
}
realCount++;
lines.Add(string.Format(
"+------T------+\tartist = {0,-30}\tcharacter = {1}",
tagDictionary[artist].tag,
tagDictionary[character].tag
));
lines.Add(string.Format(
"|{0,6}|{1,6}|",
0,
artistCount - count
));
lines.Add("+------+------+");
lines.Add(string.Format(
"|{0,6}|{1,6}|",
characterCount - count,
count
));
lines.Add("+------+------+");
lines.Add(string.Format(
"Arts by the artist to this character: {0,-6:P1}\n" +
"Arts with the char by this artist: {1,-6:P1}",
count * 1d / artistCount,
count * 1d / characterCount
));
lines.Add("=============================================");
}
await File.WriteAllLinesAsync(@"waifu-list.txt", lines, CancellationToken);
Console.WriteLine(
"{0}\twaifu-list.txt done",
DateTimeOffset.UtcNow
);
}
{
// 1й, 2й, 3й персонаж для каждого художника
var count123 = semiRawData
.GroupBy(t => t.key.artist)
.Select(t =>
{
var sum = t.Sum(t1 => t1.count);
if (sum < MIN_CHAR_COUNT_TO_CREATE_STAT)
{
return Array.Empty<double>();
}
var r = 1d / sum;
var a = t
.OrderByDescending(t1 => t1.count)
.Take(3)
.Select(t1 => t1.count * r)
.ToArray();
return a.Length switch
{
3 => a,
1 => a.Concat(new[] {0d, 0d}).ToArray(),
2 => a.Concat(new[] {0d}).ToArray(),
_ => throw new ArgumentOutOfRangeException()
};
})
.Where(x => x.Any())
.ToArray();
Console.WriteLine(
"{0}\tcount123.Length = {1,7}",
DateTimeOffset.UtcNow,
count123.Count()
);
// TODO 1й, 2й, 3й персонаж для каждого художника
var textInput = JsonConvert.SerializeObject(count123);
await File.WriteAllTextAsync(@"count123-input.json", textInput);
}
{
var rawCharFag = semiRawData
.Where(x => (x.count >= MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING) &&
(x.countP >= MIN_COUNT_PERCENT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING))
.Select(t => (fagPoint: t.countP + Math.Log10(t.count) * FAG_POINTS_LOG10_MULTIPLIER,
t.key.character))
.ToArray();
var fagCharacters = rawCharFag
.GroupBy(t => t.character)
.Select(t => (character: t.Key, sum: t.Sum(t1 => t1.fagPoint), count: t.Count()))
.OrderByDescending(t => t.sum)
.ToArray();
Console.WriteLine(
"{0}\trawCharFag.Length = {1,7}" +
"\n\t\t\t\tfagCharacters = {2,7}",
DateTimeOffset.UtcNow,
rawCharFag.Count(),
fagCharacters.Count()
);
var lines = new List<string>();
foreach (var (charId, sum, count) in fagCharacters)
{
var character = tagDictionary[charId];
lines.Add(string.Format("{0}\t{1:F2}\t{2}", character.tag, sum, count));
}
await File.WriteAllLinesAsync(@"character-fag-list.txt", lines, CancellationToken);
Console.WriteLine(
"{0}\tfag-list.txt done",
DateTimeOffset.UtcNow
);
//
var franchiseFagDic = fagCharacters
.Select(t =>
{
var (charId, fagPoint, _) = t;
if (!characterToCopyright.ContainsKey(charId))
{
return (copyright: -1, fagPoint: 0d);
}
var copyright = characterToCopyright[charId];
return (copyright, fagPoint);
})
.Where(x => x.copyright != -1)
.GroupBy(t => t.copyright)
.ToDictionary(
t => t.Key,
t => (
tag: tagDictionary[t.Key],
rawSum: t.Sum(t1 => t1.fagPoint),
limitedSum: t.Sum(t1 => Math.Min(MAX_FAG_POINTS_PER_CHARACTER, t1.fagPoint)),
charCount: t.Count()
)
);
var lines1 = franchiseFagDic
.OrderByDescending(t => t.Value.rawSum)
.Take(100)
.Select(t =>
{
var fagPoint = t.Value.rawSum;
return string.Format("{0}\t{1:F2}\t{2}", t.Value.tag.tag, fagPoint, t.Value.charCount);
})
.ToArray();
await File.WriteAllLinesAsync(@"franchise-fag-list1.txt", lines1, CancellationToken);
var lines2 = franchiseFagDic
.OrderByDescending(t => t.Value.limitedSum)
.Take(100)
.Select(t =>
{
var fagPoint = t.Value.limitedSum;
return string.Format("{0}\t{1:F2}\t{2}", t.Value.tag.tag, fagPoint, t.Value.charCount);
})
.ToArray();
await File.WriteAllLinesAsync(@"franchise-fag-list2.txt", lines2, CancellationToken);
Console.WriteLine(
"{0}\tfranchise-fag-list [both].txt done",
DateTimeOffset.UtcNow
);
}
}
#endregion
#region CalculateMeanCount123
private async Task CalculateMeanCount123()
{
List<(double[] v, int count)[]> count123Chunks;
const int CHUNK_SIZE = 1000;
const int MIN_CHUNK_SIZE = 500;
{
var text = await File.ReadAllTextAsync(@"count123-input.json");
var rawCount123 = JsonConvert.DeserializeObject<double[][]>(text);
var rnd = new Random();
var lists = new List<(double[] v, int count)[]>();
var currentList = new List<(double[] v, int count)>();
foreach (var values in rawCount123.OrderBy(_ => rnd.NextDouble()))
{
currentList.Add((values, 1));
if (currentList.Count() >= CHUNK_SIZE)
{
lists.Add(currentList
.OrderByDescending(t => t.v[0])
.ThenByDescending(t => t.v[1])
.ToArray());
currentList = new List<(double[] v, int count)>();
}
}
if (currentList.Any())
{
lists.Add(currentList
.OrderByDescending(t => t.v[0])
.ThenByDescending(t => t.v[1])
.ToArray());
}
count123Chunks = lists.ToList();
}
var fullCount = count123Chunks.Sum(t => t.Sum(t1 => t1.count));
(double[] v, int count)[] count123 = count123Chunks.First();
count123Chunks.RemoveAt(0);
var sw = Stopwatch.StartNew();
while (count123.Length > 2)
{
var len = count123.Length;
Console.WriteLine(
"\ncount123.Length = {0} + {1,4} * {2,5}",
count123.Length,
count123Chunks.Count(),
CHUNK_SIZE
);
var calculatedPairs = Enumerable
.Range(1, len - 1)
.SelectMany(index1 =>
{
return Enumerable
.Range(0, index1)
.Select(index2 => (index1, index2));
})
.AsParallel()
.Select(t =>
{
var (index1, index2) = t;
// ReSharper disable once AccessToModifiedClosure
var item1 = count123[index1];
// ReSharper disable once AccessToModifiedClosure
var item2 = count123[index2];
var sqrt = 0d;
for (var i = 0; i < 3; i++)
{
sqrt += Math.Pow(item1.v[i] - item2.v[i], 2);
}
sqrt = Math.Sqrt(sqrt / 3d);
return (index1: index1, index2: index2, sqrt);
})
.OrderBy(t => t.sqrt)
.ToArray();
var bestPair = calculatedPairs.First();
Console.WriteLine(bestPair);
if (bestPair.sqrt > 0.05)
{
break;
}
{
var item1 = count123[bestPair.index1];
var item2 = count123[bestPair.index2];
var r1 = item1.count * 1d / (item1.count + item2.count);
var r2 = 1 - r1;
var newItem = (
v: Enumerable.Range(0, 3).Select(i => item1.v[i] * r1 + item2.v[i] * r2).ToArray(),
count: item1.count + item2.count
);
Console.WriteLine("\tnew item\t{1,-5}\t{0}",
newItem.v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
newItem.count);
count123 = count123
.Select((t, index) => (t, index))
.Where(x => (x.index != bestPair.index1) && (x.index != bestPair.index2))
.Select(t => t.t)
.Concat(new[] {newItem})
.OrderByDescending(t => t.count)
.ToArray();
}
if ((count123.Length <= MIN_CHUNK_SIZE) && (count123Chunks.Any()))
{
var elapsed = sw.Elapsed;
var leftCount = count123.Length + count123Chunks.Count * CHUNK_SIZE;
var p = 1d - leftCount * 1d / fullCount;
Console.WriteLine(
"--------------------------\t{0,-7:P1} (elapsed: {1})\tETA: {2}",
p,
elapsed,
(1 - p) / p * elapsed
);
var r = 1d / count123.Sum(t => t.count);
for (int i = 0; i < Math.Min(10, count123.Length); i++)
{
var (v, count) = count123[i];
Console.WriteLine("\t{1,-5}\t{0}\t{2,6:P1}",
v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
count,
count * r
);
}
count123 = count123
.Concat(count123Chunks.First())
.ToArray();
count123Chunks.RemoveAt(0);
}
}
Console.WriteLine("==================================================");
Console.WriteLine("==================================================");
Console.WriteLine("==================================================");
var rFullCount = 1d / fullCount;
for (int i = 0; i < Math.Min(20, count123.Length); i++)
{
var (v, count) = count123[i];
Console.WriteLine("\t{1,-5}\t{0}\t{2,6:P1}",
v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
count,
count * rFullCount
);
}
}
#endregion
}
}
Comments