From 9eaf767980aa6d3f46388beeda9a37dbf11b2fb2 Mon Sep 17 00:00:00 2001 From: sentouki Date: Sun, 19 Jul 2020 20:46:52 +0200 Subject: [PATCH] - Modified ImageModel: added File Type property; - Images are now saved with the correct file type; - Implemented CheckLocalImages: already downloaded images will not be downloaded again --- ArtAPI/ArtStationAPI.cs | 3 ++- ArtAPI/DeviantArtAPI.cs | 10 ++++++--- ArtAPI/IRequestArt.cs | 9 +++++--- ArtAPI/PixivAPI.cs | 47 +++++++++++++++++++++++------------------ ArtAPI/models.cs | 4 ++++ 5 files changed, 46 insertions(+), 27 deletions(-) diff --git a/ArtAPI/ArtStationAPI.cs b/ArtAPI/ArtStationAPI.cs index 2948520..3792a52 100644 --- a/ArtAPI/ArtStationAPI.cs +++ b/ArtAPI/ArtStationAPI.cs @@ -75,7 +75,8 @@ private async Task GetAssets(string hash_id, string name) .Replace("/videos/","/images/") .Replace("/panos/", "/images/"), // workaround to download 4k images, maybe there's a better way, I might change this later Name = name, - ID = image["id"].ToString() + ID = image["id"].ToString(), + FileType = image["image_url"].ToString().Split('?')[0].Split('/').Last().Split('.')[1] }); } } diff --git a/ArtAPI/DeviantArtAPI.cs b/ArtAPI/DeviantArtAPI.cs index 7af61f3..ae01949 100644 --- a/ArtAPI/DeviantArtAPI.cs +++ b/ArtAPI/DeviantArtAPI.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Newtonsoft.Json.Linq; @@ -60,9 +61,9 @@ protected override async Task GetImagesMetadataAsync(string apiUrl) { Url = image["content"]["src"].ToString(), Name = image["title"].ToString(), - ID = image["deviationid"].ToString() - } - ); + ID = image["deviationid"].ToString(), + FileType = image["content"]["src"].ToString().Split('?')[0].Split('/').Last().Split('.')[1] // maybe not the best way but surely the the easiest one + }); } if (responseJson["has_more"].ToString() == "False") return; paginationOffset += Offset; @@ -71,6 +72,7 @@ protected override async Task GetImagesMetadataAsync(string apiUrl) public override async Task auth(string refreshToken) { + if (IsLoggedIn) return true; var data = new Dictionary() { {"grant_type", "client_credentials" }, @@ -83,6 +85,8 @@ public override async Task auth(string refreshToken) var response = await Client.PostAsync(AUTH_URL, content).ConfigureAwait(false); var jsonResponse = JObject.Parse(await response.Content.ReadAsStringAsync().ConfigureAwait(false)); if (jsonResponse["status"].ToString() == "error") return false; + if (Client.DefaultRequestHeaders.Contains("Authorization")) + Client.DefaultRequestHeaders.Remove("Authorization"); Client.DefaultRequestHeaders.Add("Authorization", $"Bearer {jsonResponse["access_token"]}"); } catch(HttpRequestException) diff --git a/ArtAPI/IRequestArt.cs b/ArtAPI/IRequestArt.cs index ebd035e..5f1d52c 100644 --- a/ArtAPI/IRequestArt.cs +++ b/ArtAPI/IRequestArt.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading; @@ -131,6 +132,7 @@ protected async Task DownloadImagesAsync() OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.DownloadCanceled, "No images to download")); Clear(); return; } + CheckLocalImages(); OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.DownloadRunning, TotalImageCount: TotalImageCount)); cts = new CancellationTokenSource(); using var ss = new SemaphoreSlim(ConcurrentTasks); @@ -175,7 +177,7 @@ protected async Task DownloadAsync(ImageModel image, string savePath) var specialChars = new List() { @"\", "/", ":", "*", "?", "\"", "<", ">", "|" }; var imageName = image.Name; specialChars.ForEach(c => imageName = imageName.Replace(c, "")); // remove all the nasty characters that can cause trouble - var imageSavePath = Path.Combine(savePath, $"{imageName}_{image.ID}.jpg"); + var imageSavePath = Path.Combine(savePath, $"{imageName}_{image.ID}.{image.FileType}"); const int tries = NumberOfDLAttempts; try { @@ -221,11 +223,12 @@ private void Clear() ImagesToDownload.Clear(); } /// - /// checks if there're already pictures saved locally and removes them from the list + /// checks if there are already pictures saved locally and removes them from the list /// private void CheckLocalImages() { - throw new NotImplementedException(); + var localImages = Directory.GetFiles(ArtistDirSavepath).Select(Path.GetFileNameWithoutExtension).ToArray(); + ImagesToDownload.RemoveAll(image => localImages.Contains($"{image.Name}_{image.ID}")); } public void CancelDownload() diff --git a/ArtAPI/PixivAPI.cs b/ArtAPI/PixivAPI.cs index 5941710..387071d 100644 --- a/ArtAPI/PixivAPI.cs +++ b/ArtAPI/PixivAPI.cs @@ -126,11 +126,8 @@ private async Task GetImageURLsWithoutLoginAsync(string illustProject) { Url = img_url["url_big"].ToString(), Name = illustDetails["title"].ToString(), - ID = img_url["url_big"].ToString() - .Split('/') - .Last() - .Replace(".png", string.Empty) - .Replace(".jpg", string.Empty) + ID = img_url["url_big"].ToString().Split('/').Last().Split('.')[0], + FileType = img_url["url_big"].ToString().Split('/').Last().Split('.')[1] }); } } @@ -143,7 +140,8 @@ private async Task GetImageURLsWithoutLoginAsync(string illustProject) { Url = illustDetails["url_big"].ToString(), Name = illustDetails["title"].ToString(), - ID = illustDetails["id"].ToString() + ID = illustDetails["id"].ToString(), + FileType = illustDetails["url_big"].ToString().Split('/').Last().Split('.')[1] }); } } @@ -157,27 +155,31 @@ private void GetImageURLsWithLogin(JObject responseJson) { foreach (var img_url in IllustDetails["meta_pages"]) { - ImagesToDownload.Add(new ImageModel() + lock (ImagesToDownload) { - Url = img_url["image_urls"]["original"].ToString(), - Name = IllustDetails["title"].ToString(), - ID = img_url["image_urls"]["original"].ToString() - .Split('/') - .Last() - .Replace(".png", string.Empty) - .Replace(".jpg", string.Empty) - }); + ImagesToDownload.Add(new ImageModel() + { + Url = img_url["image_urls"]["original"].ToString(), + Name = IllustDetails["title"].ToString(), + ID = img_url["image_urls"]["original"].ToString().Split('/').Last().Split('.')[0], + FileType = img_url["image_urls"]["original"].ToString().Split('/').Last().Split('.')[1] + }); + } } } else { - ImagesToDownload.Add(new ImageModel() + lock (ImagesToDownload) { - Url = IllustDetails["meta_single_page"]["original_image_url"].ToString(), - Name = IllustDetails["title"].ToString(), - ID = IllustDetails["id"].ToString() - }); + ImagesToDownload.Add(new ImageModel() + { + Url = IllustDetails["meta_single_page"]["original_image_url"].ToString(), + Name = IllustDetails["title"].ToString(), + ID = IllustDetails["id"].ToString(), + FileType = IllustDetails["meta_single_page"]["original_image_url"].ToString().Split('/').Last().Split('.')[1] + }); + } } } @@ -185,6 +187,7 @@ private void GetImageURLsWithLogin(JObject responseJson) public override async Task auth(string refreshToken) { + if (IsLoggedIn) return true; var clientTime = DateTime.UtcNow.ToString("s") + "+00:00"; var data = new Dictionary() { @@ -208,6 +211,8 @@ public override async Task auth(string refreshToken) var accessToken = jsonResponse["response"]["access_token"]?.ToString() ?? throw new Exception("Bad API Response"); RefreshToken = jsonResponse["response"]["refresh_token"].ToString(); + if (Client.DefaultRequestHeaders.Contains("Authorization")) + Client.DefaultRequestHeaders.Remove("Authorization"); Client.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}"); } catch (HttpRequestException) @@ -243,6 +248,8 @@ public override async Task login(string username, string password) var accessToken = jsonResponse["response"]["access_token"]?.ToString() ?? throw new Exception("Bad API Response"); RefreshToken = jsonResponse["response"]["refresh_token"].ToString(); + if (Client.DefaultRequestHeaders.Contains("Authorization")) + Client.DefaultRequestHeaders.Remove("Authorization"); Client.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}"); } catch (HttpRequestException) diff --git a/ArtAPI/models.cs b/ArtAPI/models.cs index 0b5fbd3..565a556 100644 --- a/ArtAPI/models.cs +++ b/ArtAPI/models.cs @@ -17,5 +17,9 @@ public class ImageModel /// platform-specific(Artstation, Deviantart, Pixiv...etc) ID of the image /// public string ID { get; set; } + /// + /// file type of the image (png, jpg, gif) + /// + public string FileType { get; set; } = "jpg"; } }