diff --git a/.gitattributes b/.gitattributes index a221004..5dc46e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,67 +1,3 @@ -############################################################################### -# Set default behavior to automatically normalize line endings. -############################################################################### -* text=auto - -############################################################################### -# Set default behavior for command prompt diff. -# -# This is need for earlier builds of msysgit that does not have it on by -# default for csharp files. -# Note: This is only used by command line -############################################################################### -#*.cs diff=csharp - -############################################################################### -# Set the merge driver for project and solution files -# -# Merging from the command prompt will add diff markers to the files if there -# are conflicts (Merging from VS is not affected by the settings below, in VS -# the diff markers are never inserted). Diff markers may cause the following -# file extensions to fail to load in VS. An alternative would be to treat -# these files as binary and thus will always conflict and require user -# intervention with every merge. To do so, just uncomment the entries below -############################################################################### -#*.sln merge=binary -#*.csproj merge=binary -#*.vbproj merge=binary -#*.vcxproj merge=binary -#*.vcproj merge=binary -#*.dbproj merge=binary -#*.fsproj merge=binary -#*.lsproj merge=binary -#*.wixproj merge=binary -#*.modelproj merge=binary -#*.sqlproj merge=binary -#*.wwaproj merge=binary - -############################################################################### -# behavior for image files -# -# image files are treated as binary by default. -############################################################################### -#*.jpg binary -#*.png binary -#*.gif binary - -############################################################################### -# diff behavior for common document formats -# -# Convert binary document formats to text before diffing them. This feature -# is only available from the command line. Turn it on by uncommenting the -# entries below. -############################################################################### -#*.doc diff=astextplain -#*.DOC diff=astextplain -#*.docx diff=astextplain -#*.DOCX diff=astextplain -#*.dot diff=astextplain -#*.DOT diff=astextplain -#*.pdf diff=astextplain -#*.PDF diff=astextplain -#*.rtf diff=astextplain -#*.RTF diff=astextplain - -*.exe filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.msi filter=lfs diff=lfs merge=lfs -text \ No newline at end of file +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3b9f29e..fc2565f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,46 +7,7 @@ updates: time: "04:00" open-pull-requests-limit: 10 ignore: - - dependency-name: CasCap.Common.Serialisation.Json - versions: - - 1.0.18 - - 1.0.19 - - 1.0.20 - - 1.0.22 - - 1.0.24 - - dependency-name: CasCap.Common.Caching - versions: - - 1.0.18 - - 1.0.19 - - 1.0.20 - - 1.0.22 - - 1.0.24 - - dependency-name: CasCap.Apis.GooglePhotos - versions: - - 1.0.14 - - 1.0.15 - - 1.0.16 - dependency-name: Microsoft.NET.Test.Sdk - versions: - - 16.8.3 - - 16.9.1 - - dependency-name: coverlet.collector - versions: - - 3.0.2 - - 3.0.3 + - dependency-name: Newtonsoft.Json - dependency-name: coverlet.msbuild - versions: - - 3.0.2 - - 3.0.3 - - dependency-name: SixLabors.ImageSharp - versions: - - 1.0.3 - - dependency-name: ShellProgressBar - versions: - - 5.1.0 - - dependency-name: McMaster.Extensions.CommandLineUtils - versions: - - 3.1.0 - - dependency-name: McMaster.Extensions.Hosting.CommandLine - versions: - - 3.1.0 + - dependency-name: coverlet.collector diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7701f1c..6aae5ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,186 +4,32 @@ on: workflow_dispatch: inputs: BuildConfiguration: + type: choice description: Build Configuration required: true default: Release + options: + - Release + - Debug + PublishPreview: + type: string + description: Publish preview branch? + required: true + default: "false" push: + branches-ignore: + - "preview/**" paths-ignore: - - '.azure-pipelines/**' - - LICENSE - - README.md + - ".azure-pipelines/**" + - LICENSE + - README.md pull_request: branches: [main] types: [opened, synchronize, reopened] jobs: - build: - #no point using matrix build as "Container operations are only supported on Linux runners" - # strategy: - # matrix: - # os: [ubuntu-latest,windows-latest] - # runs-on: ${{matrix.os}} - runs-on: ubuntu-latest - outputs: - SemVer: ${{ steps.gitversion.outputs.SemVer }} - - steps: - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: .NET Core 3.1.x SDK - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 3.1.x - - - name: .NET 5.x SDK - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 5.0.x - - # - name: gitversion install - # uses: gittools/actions/gitversion/setup@v0.9.7 - # with: - # versionSpec: 5.x - - # #https://gitversion.net/docs/usage/ - # - name: gitversion determine version - # id: gitversionOLD - # uses: gittools/actions/gitversion/execute@v0.9.7 - # with: - # useConfigFile: true - - - name: gitversion - id: gitversion - run: | - dotnet tool update -g GitVersion.Tool - $GitVersion = dotnet-gitversion ${{ github.workspace }} /nofetch | ConvertFrom-Json - echo "SemVer=$($GitVersion.SemVer)" - echo "::set-output name=SemVer::$($GitVersion.SemVer)" - shell: pwsh - - - name: dotnet restore - run: dotnet restore --verbosity minimal --configfile nuget.config - - - name: dotnet build - run: dotnet build -c Release --nologo --no-restore -p:Version='${{ steps.gitversion.outputs.SemVer }}' -p:SourceRevisionId=${{ github.sha }} - - - name: dotnet test - run: dotnet test -c Release --nologo --no-restore --no-build -p:CollectCoverage=true -p:CoverletOutputFormat=lcov -p:CoverletOutput=${{ github.workspace }}/coverage/ - - - name: code coverage - coveralls (1 of 3) - .NET Core 3.1.x - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - path-to-lcov: ${{ github.workspace }}/coverage/coverage.netcoreapp3.1.info - flag-name: run-netcoreapp3.1 - parallel: true - - - name: code coverage - coveralls (2 of 3) - .NET 5.x SDK - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - path-to-lcov: ${{ github.workspace }}/coverage/coverage.net5.0.info - flag-name: run-net5.0 - parallel: true - - - name: code coverage - coveralls (3 of 3) - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - parallel-finished: true - - # - name: code coverage - report generator - # uses: danielpalme/ReportGenerator-GitHub-Action@4.5.8 - # if: runner.OS == 'Linux' - # with: - # reports: ./coverage/coverage.*.info - # targetdir: ./coveragereport - # reporttypes: lcov - # tag: ${{ github.run_number }}_${{ github.run_id }} - - - name: reportgenerator - if: runner.OS == 'Linux' - run: | - dotnet tool update -g dotnet-reportgenerator-globaltool - reportgenerator -reports:./coverage/coverage.*.info \ - -targetdir:./coveragereport \ - -reporttypes:lcov \ - -tag:${{ github.run_number }}_${{ github.run_id }} - - - name: code coverage - upload-artifact - uses: actions/upload-artifact@v2 - if: runner.OS == 'Linux' - with: - name: coveragereport - path: coveragereport - - - name: dotnet pack - run: dotnet pack -c Release --nologo --no-build --include-symbols -p:Version='${{ steps.gitversion.outputs.SemVer }}' - if: github.ref == 'refs/heads/main' && runner.OS == 'Linux' - - - name: dotnet push (nuget) - run: dotnet nuget push ${{ github.workspace }}/src/**/*.nupkg --skip-duplicate -k ${{ secrets.NUGET_API_KEY }} -s https://api.nuget.org/v3/index.json - if: github.ref == 'refs/heads/main' - - - name: dotnet push (github) - run: | - dotnet tool update -g gpr - gpr push ${{ github.workspace }}/src/**/*.nupkg -k ${{ secrets.GITHUB_TOKEN }} - if: github.ref == 'refs/heads/main' - - - name: create-release ${{ steps.gitversion.outputs.SemVer }} - uses: actions/create-release@v1 #todo: this is deprecated, replace with another action later...? - if: github.ref == 'refs/heads/main' - with: - tag_name: ${{ steps.gitversion.outputs.SemVer }} - release_name: Release ${{ steps.gitversion.outputs.SemVer }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - SonarQube: - runs-on: windows-latest - steps: - # below composite action does not work, github doesn't support actions inside composite actions :/ - # https://github.com/actions/runner/issues/646 - # - uses: ./.github/actions/sonarqube - # with: - # SonarToken: ${{ secrets.SONAR_TOKEN }} - - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 1.11 - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Cache SonarCloud packages - uses: actions/cache@v1 - with: - path: ~\sonar\cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Cache SonarCloud scanner - id: cache-sonar-scanner - uses: actions/cache@v1 - with: - path: .\.sonar\scanner - key: ${{ runner.os }}-sonar-scanner - restore-keys: ${{ runner.os }}-sonar-scanner - - name: Install SonarCloud scanner - if: steps.cache-sonar-scanner.outputs.cache-hit != 'true' - shell: powershell - run: | - New-Item -Path .\.sonar\scanner -ItemType Directory - dotnet tool update dotnet-sonarscanner --tool-path .\.sonar\scanner - - name: Build and analyze - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - shell: powershell - run: | - .\.sonar\scanner\dotnet-sonarscanner begin /k:"${{ github.actor }}_${{ github.event.repository.name }}" /o:"${{ github.actor }}" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" - dotnet build -c Release - .\.sonar\scanner\dotnet-sonarscanner end /d:sonar.login="${{ secrets.SONAR_TOKEN }}" \ No newline at end of file + ci: + uses: f2calv/gha-workflows/.github/workflows/dotnet-publish-nuget-v1.yml@main + with: + BuildConfiguration: ${{ github.event.inputs.BuildConfiguration }} + PublishPreview: ${{ github.event.inputs.PublishPreview }} \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props index c25f129..1af1711 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,10 +2,11 @@ CasCap - 9.0 + 10.0 255ba51e-c46d-40c4-9420-80d485ea10fe + enable @@ -34,8 +35,10 @@ - - IDE1006 + + + + IDE1006;IDE0079;IDE0042 @@ -43,7 +46,7 @@ - + diff --git a/GitVersion.yml b/GitVersion.yml index 81258f6..666e775 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,9 +1,15 @@ mode: MainLine branches: main: + regex: ^main$ tag: '' feature: - tag: ci + regex: ^features?[/-] + tag: useBranchName + preview: + regex: ^preview?[/-] + tag: preview-{BranchName} + source-branches: ['main'] ignore: sha: [] -merge-message-formats: {} \ No newline at end of file +merge-message-formats: {} diff --git a/README.md b/README.md index d003769..cb55d1c 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,12 @@ ![CI](https://github.com/f2calv/CasCap.GooglePhotosCli/actions/workflows/ci.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/f2calv/CasCap.GooglePhotosCli/badge.svg?branch=main)](https://coveralls.io/github/f2calv/CasCap.GooglePhotosCli?branch=main) [![SonarCloud Coverage](https://sonarcloud.io/api/project_badges/measure?project=f2calv_CasCap.GooglePhotosCli&metric=code_smells)](https://sonarcloud.io/component_measures/metric/code_smells/list?id=f2calv_CasCap.GooglePhotosCli) [![Nuget][cascap.apis.googlephotoscli-badge]][cascap.apis.googlephotoscli-url] -This is an _unofficial_ Google Photos CLI which can be installed as a .NET Global Tool +This is an _unofficial_ Google Photos CLI which can be installed as a .NET Global Tool. Google Photos CLI is an _unofficial_ utility which leverages the [CasCap.Apis.GooglePhotos](https://github.com/f2calv/CasCap.Apis.GooglePhotos) library to perform common and helpful operations against the media items held in your Google Photos account. +If you find this tool of use then please give it a thumbs-up by giving this repository a :star: ... :wink: + Key functionality; - Media item upload @@ -25,7 +27,7 @@ Key functionality; The Google Photos CLI is distributed as a [.NET Core Global Tool](https://docs.microsoft.com/en-us/dotnet/core/tools/global-tools), to install the tool follow these steps; - Follow [these instructions](https://github.com/f2calv/CasCap.Apis.GooglePhotos#google-photos-api-set-up) to set-up OAuth login details. -- Download and install either the [.NET Core 3.1 SDK](https://dotnet.microsoft.com/download/dotnet-core/3.1) or [.NET 5.0 SDK](https://dotnet.microsoft.com/download/dotnet/5.0). +- Download and install either the [.NET Core 3.1 SDK](https://dotnet.microsoft.com/download/dotnet-core/3.1) or [.NET 6.0 SDK](https://dotnet.microsoft.com/download/dotnet/6.0). - From a command line shell install the tool `dotnet tool update --global googlephotos` Now check the tool is installed by entering `googlephotos` at a shell. diff --git a/global.json b/global.json index 6c93f2d..7b83888 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - //"allowPrerelease": false + "allowPrerelease": false } } \ No newline at end of file diff --git a/nuget.config b/nuget.config index 0a30c61..2a82514 100644 --- a/nuget.config +++ b/nuget.config @@ -4,5 +4,6 @@ + \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli.Tests/CasCap.GooglePhotosCli.Tests.csproj b/src/CasCap.GooglePhotosCli.Tests/CasCap.GooglePhotosCli.Tests.csproj index 1337986..d442b8e 100644 --- a/src/CasCap.GooglePhotosCli.Tests/CasCap.GooglePhotosCli.Tests.csproj +++ b/src/CasCap.GooglePhotosCli.Tests/CasCap.GooglePhotosCli.Tests.csproj @@ -2,26 +2,26 @@ netcoreapp3.1 - $(TargetFrameworks);net5.0 + $(TargetFrameworks);net6.0 - - - - - - + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/CasCap.GooglePhotosCli.Tests/Tests/TestBase.cs b/src/CasCap.GooglePhotosCli.Tests/Tests/TestBase.cs index ced3a73..810baea 100644 --- a/src/CasCap.GooglePhotosCli.Tests/Tests/TestBase.cs +++ b/src/CasCap.GooglePhotosCli.Tests/Tests/TestBase.cs @@ -3,38 +3,37 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Xunit.Abstractions; -namespace CasCap.GooglePhotosCli.Tests +namespace CasCap.GooglePhotosCli.Tests; + +public abstract class TestBase { - public abstract class TestBase - { - protected ILogger _logger; + protected ILogger _logger; - protected GooglePhotosService _googlePhotosSvc; - protected DiskCacheService _diskCacheSvc; + protected GooglePhotosService _googlePhotosSvc; + protected DiskCacheService _diskCacheSvc; - public TestBase(ITestOutputHelper output) - { - var configuration = new ConfigurationBuilder() - .AddJsonFile($"appsettings.Test.json", optional: false, reloadOnChange: true) - .AddUserSecrets()//for local testing - .AddEnvironmentVariables()//for CI testing - .Build(); + public TestBase(ITestOutputHelper output) + { + var configuration = new ConfigurationBuilder() + .AddJsonFile($"appsettings.Test.json", optional: false, reloadOnChange: true) + .AddUserSecrets()//for local testing + .AddEnvironmentVariables()//for CI testing + .Build(); - //initiate ServiceCollection w/logging - var services = new ServiceCollection() - .AddSingleton(configuration) - .AddXUnitLogging(output); + //initiate ServiceCollection w/logging + var services = new ServiceCollection() + .AddSingleton(configuration) + .AddXUnitLogging(output); - _logger = ApplicationLogging.LoggerFactory.CreateLogger(); + _logger = ApplicationLogging.LoggerFactory.CreateLogger(); - //add services - services.AddGooglePhotos(); - services.AddSingleton(); + //add services + services.AddGooglePhotos(); + services.AddSingleton(); - //retrieve services - var serviceProvider = services.BuildServiceProvider(); - _googlePhotosSvc = serviceProvider.GetRequiredService(); - _diskCacheSvc = serviceProvider.GetRequiredService(); - } + //retrieve services + var serviceProvider = services.BuildServiceProvider(); + _googlePhotosSvc = serviceProvider.GetRequiredService(); + _diskCacheSvc = serviceProvider.GetRequiredService(); } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli.Tests/Tests/UnitTest1.cs b/src/CasCap.GooglePhotosCli.Tests/Tests/UnitTest1.cs index 6a77c4a..3c1e519 100644 --- a/src/CasCap.GooglePhotosCli.Tests/Tests/UnitTest1.cs +++ b/src/CasCap.GooglePhotosCli.Tests/Tests/UnitTest1.cs @@ -1,28 +1,25 @@ -using System; using System.Diagnostics; -using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; -namespace CasCap.GooglePhotosCli.Tests +namespace CasCap.GooglePhotosCli.Tests; + +public class CliTests : TestBase { - public class CliTests : TestBase - { - public CliTests(ITestOutputHelper output) : base(output) { } + public CliTests(ITestOutputHelper output) : base(output) { } - [Fact] - public async Task Test1() + [Fact] + public async Task Test1() + { + //todo: how best to test a command line application? + try + { + _ = await _googlePhotosSvc.CreateAlbumAsync("test"); + } + catch (Exception ex) { - //todo: how best to test a command line application? - try - { - _ = await _googlePhotosSvc.CreateAlbumAsync("test"); - } - catch (Exception ex) - { - Debug.WriteLine(ex); - Debugger.Break(); - } - Assert.True(true);//assert true regardless of actual outcome, will add full tests later + Debug.WriteLine(ex); + Debugger.Break(); } + Assert.True(true);//assert true regardless of actual outcome, will add full tests later } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/CasCap.GooglePhotosCli.csproj b/src/CasCap.GooglePhotosCli/CasCap.GooglePhotosCli.csproj index ff939b9..998a175 100644 --- a/src/CasCap.GooglePhotosCli/CasCap.GooglePhotosCli.csproj +++ b/src/CasCap.GooglePhotosCli/CasCap.GooglePhotosCli.csproj @@ -3,7 +3,7 @@ Exe netcoreapp3.1 - $(TargetFrameworks);net5.0 + $(TargetFrameworks);net6.0 googlephotos @@ -15,15 +15,15 @@ - - + + - - - - - - + + + + + + \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Albums.cs b/src/CasCap.GooglePhotosCli/Commands/Albums.cs index 40a7fac..182a2db 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Albums.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Albums.cs @@ -3,216 +3,210 @@ using CasCap.Services; using McMaster.Extensions.CommandLineUtils; using ShellProgressBar; -using System; -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command(Description = "Manage your media library albums.")] +[Subcommand(typeof(Add))] +[Subcommand(typeof(List))] +[Subcommand(typeof(Sync))] +[Subcommand(typeof(Download))] +internal class Albums : CommandBase { - [Command(Description = "Manage your media library albums.")] - [Subcommand(typeof(Add))] - [Subcommand(typeof(List))] - [Subcommand(typeof(Sync))] - [Subcommand(typeof(Download))] - internal class Albums : CommandBase - { - public Albums(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + public Albums(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await Task.Delay(0); - //await base.OnExecuteAsync(app); - //_console.Error.WriteLine("You must specify an action. See --help for more details."); - app.ShowHelp(); - return 1; - } + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await Task.Delay(0); + //await base.OnExecuteAsync(app); + //_console.Error.WriteLine("You must specify an action. See --help for more details."); + app.ShowHelp(); + return 1; + } - [Command(Description = "List existing album details.")] - class List : CommandBase - { - public List(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + [Command(Description = "List existing album details.")] + class List : CommandBase + { + public List(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - [Option("--duplicates", Description = "Show only duplicate albums by title.")] - public bool duplicatesOnly { get; } + [Option("--duplicates", Description = "Show only duplicate albums by title.")] + public bool duplicatesOnly { get; } - public async override Task OnExecuteAsync(CommandLineApplication app) + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); + //todo: use progress bars to display album batch retrieval better? + //output data to CSV/HTML/? if HTML launch a browser? + var albums = await _googlePhotosSvc.GetAlbumsAsync(); + if (albums.IsNullOrEmpty()) { - await base.OnExecuteAsync(app); - //todo: use progress bars to display album batch retrieval better? - //output data to CSV/HTML/? if HTML launch a browser? - var albums = await _googlePhotosSvc.GetAlbumsAsync(); - if (albums.IsNullOrEmpty()) - { - _console.WriteLine("Sorry, no album data available..."); - return 0; - } - if (duplicatesOnly) - { - var duplicates = GetAlbumDuplicates(albums); - _console.WriteLine($"{albums.Count} album(s) found, {duplicates.Count} duplicate album(s) detected."); - if (duplicates.IsNullOrEmpty()) - return 0; - DisplayAlbums(duplicates); - } - else - DisplayAlbums(albums); - + _console.WriteLine("Sorry, no album data available..."); return 0; } + if (duplicatesOnly) + { + var duplicates = GetAlbumDuplicates(albums); + _console.WriteLine($"{albums.Count} album(s) found, {duplicates.Count} duplicate album(s) detected."); + if (duplicates.IsNullOrEmpty()) + return 0; + DisplayAlbums(duplicates); + } + else + DisplayAlbums(albums); + + return 0; } + } - [Command(Description = "Add new album.")] - class Add : CommandBase - { - public Add(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + [Command(Description = "Add new album.")] + class Add : CommandBase + { + public Add(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - [Required] - [Option("-t|--title", Description = "Album title")] - public string title { get; } + [Required] + [Option("-t|--title", Description = "Album title")] + public string title { get; } - //[Option("--allowduplicate", Description = "Allow duplicate album creation by title.")] - //public bool allowDuplicate { get; } + //[Option("--allowduplicate", Description = "Allow duplicate album creation by title.")] + //public bool allowDuplicate { get; } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); - var album = await _googlePhotosSvc.GetOrCreateAlbumAsync(title); - if (album is object) - _console.WriteLine($"Created OR retrieved '{album.title}' with id '{album.id}'"); - else - _console.WriteLine($"Sorry unable to create album '{album.title}', maybe you don't have the right permissions?"); - return 0; - } + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); + var album = await _googlePhotosSvc.GetOrCreateAlbumAsync(title); + if (album is object) + _console.WriteLine($"Created OR retrieved '{album.title}' with id '{album.id}'"); + else + _console.WriteLine($"Sorry unable to create album '{album.title}', maybe you don't have the right permissions?"); + return 0; } + } - [Command(Description = "Refresh local album cache.")] - class Sync : CommandBase - { - public Sync(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + [Command(Description = "Refresh local album cache.")] + class Sync : CommandBase + { + public Sync(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); - //todo: use progress bars to display album batch retrieval better? - await SyncAlbums(); - //todo: improve the UI output here - _console.WriteLine($"Sync completed."); - return 0; - } + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); + //todo: use progress bars to display album batch retrieval better? + await SyncAlbums(); + //todo: improve the UI output here + _console.WriteLine($"Sync completed."); + return 0; } + } - [Command(Description = "Download album media items.")] - class Download : CommandBase - { - public Download(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + [Command(Description = "Download album media items.")] + class Download : CommandBase + { + public Download(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - [Required] - [Option("-t|--title", Description = "Album title")] - public string title { get; } + [Required] + [Option("-t|--title", Description = "Album title")] + public string title { get; } - [Option("-o|--output", Description = "Output path")] - public string outputPath { get; } + [Option("-o|--output", Description = "Output path")] + public string outputPath { get; } - [Option("-y|--yes", Description = "Assume Yes.")]//todo: improve description - public bool AutoConfirm { get; } + [Option("-y|--yes", Description = "Assume Yes.")]//todo: improve description + public bool AutoConfirm { get; } - [Option("-w|--maxwidth", Description = "Scale the image with this max width, preserving the aspect ratio.")] - public int? maxWidth { get; } + [Option("-w|--maxwidth", Description = "Scale the image with this max width, preserving the aspect ratio.")] + public int? maxWidth { get; } - [Option("-h|--maxheight", Description = "Scale the image with this max height, preserving the aspect ratio.")] - public int? maxHeight { get; } + [Option("-h|--maxheight", Description = "Scale the image with this max height, preserving the aspect ratio.")] + public int? maxHeight { get; } - [Option("--crop", Description = "Crop the image to the exact values of max width and max height.")] - public bool crop { get; } + [Option("--crop", Description = "Crop the image to the exact values of max width and max height.")] + public bool crop { get; } - [Option("--exif", Description = "Download the image retaining all the EXIF metadata except the location metadata.")] - public bool exif { get; } + [Option("--exif", Description = "Download the image retaining all the EXIF metadata except the location metadata.")] + public bool exif { get; } - [Option("--overwrite", Description = "Re-download the media item even if it exists locally.")] - public bool overwrite { get; } + [Option("--overwrite", Description = "Re-download the media item even if it exists locally.")] + public bool overwrite { get; } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); - var path = AppDomain.CurrentDomain.BaseDirectory; - if (outputPath is object) path = outputPath; - if (!path.EndsWith(Path.DirectorySeparatorChar)) path += Path.DirectorySeparatorChar; - if (!Directory.Exists(path)) - if (!AutoConfirm && !Prompt.GetYesNo($"Directory '{path}' does not exist, create?", true)) - return 0; - Directory.CreateDirectory(path);//create the folder if doesn't exist + var path = AppDomain.CurrentDomain.BaseDirectory; + if (outputPath is object) path = outputPath; + if (!path.EndsWith(Path.DirectorySeparatorChar)) path += Path.DirectorySeparatorChar; + if (!Directory.Exists(path)) + if (!AutoConfirm && !Prompt.GetYesNo($"Directory '{path}' does not exist, create?", true)) + return 0; + Directory.CreateDirectory(path);//create the folder if doesn't exist - var rootPath = Path.GetFullPath(path); + var rootPath = Path.GetFullPath(path); - var album = await _googlePhotosSvc.GetAlbumByTitleAsync(title); - if (album is null) - { - _console.WriteLine($"Album with title '{title}' not found!"); - return 0; - } - var mediaItems = await _googlePhotosSvc.GetMediaItemsByAlbumAsync(album.id); - if (mediaItems.IsNullOrEmpty()) - { - _console.WriteLine($"Album with title '{title}' exists, but contains no media items!"); - return 0; - } - //note: if the album has thousands of photos and they attempt to dl them all, after 1 hour the earlier baseUrl's will have expired... + var album = await _googlePhotosSvc.GetAlbumByTitleAsync(title); + if (album is null) + { + _console.WriteLine($"Album with title '{title}' not found!"); + return 0; + } + var mediaItems = await _googlePhotosSvc.GetMediaItemsByAlbumAsync(album.id); + if (mediaItems.IsNullOrEmpty()) + { + _console.WriteLine($"Album with title '{title}' exists, but contains no media items!"); + return 0; + } + //note: if the album has thousands of photos and they attempt to dl them all, after 1 hour the earlier baseUrl's will have expired... - var allFileInfos = GetFiles(rootPath); + var allFileInfos = GetFiles(rootPath); - var items = new List(mediaItems.Count); - foreach (var mediaItem in mediaItems) + var items = new List(mediaItems.Count); + foreach (var mediaItem in mediaItems) + { + //check if the file already exists locally + var fileInfo = allFileInfos.FirstOrDefault(p => Path.GetFileName(p.FullName).Equals(mediaItem.filename, StringComparison.OrdinalIgnoreCase)); + var mi = new MyMediaFileItem { mediaItem = mediaItem, albums = new[] { title } }; + if (fileInfo is object) + mi.relPath = GetRelPath(rootPath, fileInfo); + else + mi.relPath = Path.Combine(rootPath, mediaItem.filename); + if (overwrite || fileInfo is null) { - //check if the file already exists locally - var fileInfo = allFileInfos.FirstOrDefault(p => Path.GetFileName(p.FullName).Equals(mediaItem.filename, StringComparison.OrdinalIgnoreCase)); - var mi = new MyMediaFileItem { mediaItem = mediaItem, albums = new[] { title } }; - if (fileInfo is object) - mi.relPath = GetRelPath(rootPath, fileInfo); - else - mi.relPath = Path.Combine(rootPath, mediaItem.filename); - if (overwrite || fileInfo is null) - { - //todo: check filename is unique - might have to create a txt file in the directory of known renames, otherwise file1.jpg, file1(1).jpg, will recurse - items.Add(mi); - } + //todo: check filename is unique - might have to create a txt file in the directory of known renames, otherwise file1.jpg, file1(1).jpg, will recurse + items.Add(mi); } - if (items.IsNullOrEmpty()) - { - _console.WriteLine($"No new media items exist. Use argument --{nameof(overwrite)} to force a re-download/overwrite of all media items."); - return 0; - } - //todo: display summary table of what will be downloaded? - - var dtStart = DateTime.UtcNow; - var estimatedDuration = TimeSpan.FromMilliseconds(items.Count * 2_000);//set gu-estimatedDuration + } + if (items.IsNullOrEmpty()) + { + _console.WriteLine($"No new media items exist. Use argument --{nameof(overwrite)} to force a re-download/overwrite of all media items."); + return 0; + } + //todo: display summary table of what will be downloaded? - pbar = new ProgressBar(items.Count, $"Downloading {items.Count} media item(s)...", pbarOptions) - { - EstimatedDuration = estimatedDuration - }; + var dtStart = DateTime.UtcNow; + var estimatedDuration = TimeSpan.FromMilliseconds(items.Count * 2_000);//set gu-estimatedDuration - foreach (var item in items)//in what file order do we download big albums? - { - if (item.mediaItem.syncDate < DateTime.UtcNow.AddHours(-1)) - throw new Exception($"mediaitem has expired, refresh the item...");//todo: handle this better - - //todo: add child progress bar and HttpClient download progress meter https://github.com/dotnet/runtime/issues/16681 - var bytes = await _googlePhotosSvc.DownloadBytes(item.mediaItem, maxWidth, maxHeight, crop, exif); - var fullPath = Path.Combine(rootPath, item.relPath); - File.WriteAllBytes(fullPath, bytes); - item.fileInfo = new FileInfo(fullPath); - pbar.Tick(); - } - pbar.Dispose(); + pbar = new ProgressBar(items.Count, $"Downloading {items.Count} media item(s)...", pbarOptions) + { + EstimatedDuration = estimatedDuration + }; - //todo: improve the UI output here - _console.WriteLine($"Downloaded {items.Count} media item(s) to {rootPath}, {items.Sum(p => p.fileInfo.Length).GetSizeInMB():#,##0.0} MB."); - return 0; + foreach (var item in items)//in what file order do we download big albums? + { + if (item.mediaItem.syncDate < DateTime.UtcNow.AddHours(-1)) + throw new Exception($"mediaitem has expired, refresh the item...");//todo: handle this better + + //todo: add child progress bar and HttpClient download progress meter https://github.com/dotnet/runtime/issues/16681 + var bytes = await _googlePhotosSvc.DownloadBytes(item.mediaItem, maxWidth, maxHeight, crop, exif); + var fullPath = Path.Combine(rootPath, item.relPath); + File.WriteAllBytes(fullPath, bytes); + item.fileInfo = new FileInfo(fullPath); + pbar.Tick(); } + pbar.Dispose(); + + //todo: improve the UI output here + _console.WriteLine($"Downloaded {items.Count} media item(s) to {rootPath}, {items.Sum(p => p.fileInfo.Length).GetSizeInMB():#,##0.0} MB."); + return 0; } } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Base/CommandBase.cs b/src/CasCap.GooglePhotosCli/Commands/Base/CommandBase.cs index 1c0c1ab..0e01e1a 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Base/CommandBase.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Base/CommandBase.cs @@ -5,484 +5,478 @@ using CasCap.ViewModels; using McMaster.Extensions.CommandLineUtils; using ShellProgressBar; -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; -using System.Linq; using System.Runtime.CompilerServices; -using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[HelpOption("--help")] +internal abstract class CommandBase { - [HelpOption("--help")] - internal abstract class CommandBase - { - readonly string[] skipFileNames = new[] { "color_pop.jpg", "effects.jpg" }; - //string[] skipFileNames = new string[] { }; + readonly string[] skipFileNames = new[] { "color_pop.jpg", "effects.jpg" }; + //string[] skipFileNames = new string[] { }; + + protected string duplicateFolder = null; - protected string duplicateFolder = null; + List allMediaItems { get; set; } = new();//todo: make this a private and always use the dictionary values instead + protected Dictionary dMediaItems { get; set; } = new();//this is the primary reference to mediaItem - List allMediaItems { get; set; } = new();//todo: make this a private and always use the dictionary values instead - protected Dictionary dMediaItems { get; set; } = new();//this is the primary reference to mediaItem + protected List allAlbums { get; set; } + Dictionary dAlbums { get; set; } = new(); - protected List allAlbums { get; set; } - Dictionary dAlbums { get; set; } = new(); + Dictionary> dMediaItemsByAlbum { get; set; } = new();//reference to main MediaItem + Dictionary> dMediaItemsByCategory = new();//reference to main MediaItem - Dictionary> dMediaItemsByAlbum { get; set; } = new();//reference to main MediaItem - Dictionary> dMediaItemsByCategory = new();//reference to main MediaItem + protected ProgressBar pbar; + protected ChildProgressBar childPBar; - protected ProgressBar pbar; - protected ChildProgressBar childPBar; + protected readonly IConsole _console; + protected readonly DiskCacheService _diskCacheSvc; + protected readonly GooglePhotosService _googlePhotosSvc; + + public CommandBase(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) + { + _console = console; + _diskCacheSvc = diskCacheSvc; + _diskCacheSvc.CacheRoot = _fileDataStoreFullPathOverride; + _googlePhotosSvc = googlePhotosSvc; + _googlePhotosSvc.PagingEvent += GooglePhotosSvc_PagingEvent; + } + + public virtual void GooglePhotosSvc_PagingEvent(object sender, PagingEventArgs e) + { + var str = $"Page {e.pageNumber}\t{e.recordCount}\t+{e.pageSize}"; + if (e.minDate.HasValue && e.maxDate.HasValue) + str += $"\t{e.minDate.Value:yyyy-MM-dd HH:mm} to {e.maxDate.Value:yyyy-MM-dd HH:mm} ({e.minDate.Value.GetTimeDifference(e.maxDate.Value)})"; + _console.WriteLine(str); + } - protected readonly IConsole _console; - protected readonly DiskCacheService _diskCacheSvc; - protected readonly GooglePhotosService _googlePhotosSvc; + public virtual async Task OnExecuteAsync(CommandLineApplication app) + { + await FastLogin(); + await ReadConfig(); + return 0; + } - public CommandBase(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) + async Task FastLogin() + { + var hasLoggedInBefore = File.Exists(_optionsFilePath); + if (hasLoggedInBefore) { - _console = console; - _diskCacheSvc = diskCacheSvc; - _diskCacheSvc.CacheRoot = _fileDataStoreFullPathOverride; - _googlePhotosSvc = googlePhotosSvc; - _googlePhotosSvc.PagingEvent += GooglePhotosSvc_PagingEvent; + var str = File.ReadAllText(_optionsFilePath); + _options = str.FromJSON(); } - public virtual void GooglePhotosSvc_PagingEvent(object sender, PagingEventArgs e) + if (!hasLoggedInBefore || !await _googlePhotosSvc.LoginAsync(_options)) { - var str = $"Page {e.pageNumber}\t{e.recordCount}\t+{e.pageSize}"; - if (e.minDate.HasValue && e.maxDate.HasValue) - str += $"\t{e.minDate.Value:yyyy-MM-dd HH:mm} to {e.maxDate.Value:yyyy-MM-dd HH:mm} ({e.minDate.Value.GetTimeDifference(e.maxDate.Value)})"; - _console.WriteLine(str); + //_console.WriteLine("Please call login first..."); + await Login(); + return; } - public virtual async Task OnExecuteAsync(CommandLineApplication app) + _userPath = Path.Combine(_options.FileDataStoreFullPathOverride, _options.User); + if (!Directory.Exists(_userPath)) { - await FastLogin(); - await ReadConfig(); - return 0; + Directory.CreateDirectory(_userPath); + _console.WriteLine($"creating user folder {_userPath}"); } + _diskCacheSvc.CacheRoot = _userPath; + } - async Task FastLogin() + async Task Login() + { + //_console.WriteLine(_optionsFilePath); + var hasLoggedInBefore = File.Exists(_optionsFilePath); + if (hasLoggedInBefore) { - var hasLoggedInBefore = File.Exists(_optionsFilePath); - if (hasLoggedInBefore) - { - var str = File.ReadAllText(_optionsFilePath); - _options = str.FromJSON(); - } - - if (!hasLoggedInBefore || !await _googlePhotosSvc.LoginAsync(_options)) - { - //_console.WriteLine("Please call login first..."); - await Login(); - return; - } - - _userPath = Path.Combine(_options.FileDataStoreFullPathOverride, _options.User); - if (!Directory.Exists(_userPath)) - { - Directory.CreateDirectory(_userPath); - _console.WriteLine($"creating user folder {_userPath}"); - } - _diskCacheSvc.CacheRoot = _userPath; + hasLoggedInBefore = Prompt.GetYesNo("You have logged-in before, use same log-in details?", true, _promptColor, _promptBgColor); + //_console.WriteLine("You have logged-in before, using persisted authentication details..."); + //hasLoggedInBefore = true; } + var saveDetails = false; - async Task Login() + if (!hasLoggedInBefore) { - //_console.WriteLine(_optionsFilePath); - var hasLoggedInBefore = File.Exists(_optionsFilePath); - if (hasLoggedInBefore) + _options = new GooglePhotosOptions { - hasLoggedInBefore = Prompt.GetYesNo("You have logged-in before, use same log-in details?", true, _promptColor, _promptBgColor); - //_console.WriteLine("You have logged-in before, using persisted authentication details..."); - //hasLoggedInBefore = true; - } - var saveDetails = false; - - if (!hasLoggedInBefore) - { - _options = new GooglePhotosOptions - { - User = Prompt.GetString("What is your email?", promptColor: _promptColor, promptBgColor: _promptBgColor), - //todo: validate the above input? - - ClientId = Prompt.GetString("What is your ClientId?", promptColor: _promptColor, promptBgColor: _promptBgColor), - //todo: validate the above input? + User = Prompt.GetString("What is your email?", promptColor: _promptColor, promptBgColor: _promptBgColor), + //todo: validate the above input? - ClientSecret = Prompt.GetString("What is your ClientSecret?", promptColor: _promptColor, promptBgColor: _promptBgColor), - //todo: validate the above input? + ClientId = Prompt.GetString("What is your ClientId?", promptColor: _promptColor, promptBgColor: _promptBgColor), + //todo: validate the above input? - //todo: create this extension in mcmasterlib - PromptGetStringArray() ? - //options.Scopes = new[] { GooglePhotosScope.ReadOnly }; - Scopes = new[] { GooglePhotosScope.Access, GooglePhotosScope.Sharing }, - FileDataStoreFullPathOverride = _fileDataStoreFullPathOverride - }; + ClientSecret = Prompt.GetString("What is your ClientSecret?", promptColor: _promptColor, promptBgColor: _promptBgColor), + //todo: validate the above input? - saveDetails = Prompt.GetYesNo("Persist these log-in details for next time?", false, _promptColor, _promptBgColor); - } - else - { - var str = File.ReadAllText(_optionsFilePath); - _options = str.FromJSON(); - } + //todo: create this extension in mcmasterlib - PromptGetStringArray() ? + //options.Scopes = new[] { GooglePhotosScope.ReadOnly }; + Scopes = new[] { GooglePhotosScope.Access, GooglePhotosScope.Sharing }, + FileDataStoreFullPathOverride = _fileDataStoreFullPathOverride + }; - var success = await _googlePhotosSvc.LoginAsync(_options); - if (success) - { - if (!hasLoggedInBefore && saveDetails) - File.WriteAllText(_optionsFilePath, _options.ToJSON()); - _console.WriteLine("now logged in!! :)"); - } - else - _console.WriteLine("login failed :)"); + saveDetails = Prompt.GetYesNo("Persist these log-in details for next time?", false, _promptColor, _promptBgColor); } - - protected static void DisplayAlbums(List albums) + else { - var headers = new[] { new ColumnHeader("#"), new ColumnHeader("Title"), new ColumnHeader("Items", Alignment.Right), new ColumnHeader("Id") }; - var table = new Table(headers) { Config = TableConfiguration.Markdown() }; - var i = 1; - foreach (var album in albums) - { - table.AddRow(i, album.title, album.mediaItemsCount, album.id); - i++; - } - Console.Write(table.ToString()); + var str = File.ReadAllText(_optionsFilePath); + _options = str.FromJSON(); } - protected ProgressBarOptions pbarOptions { get; set; } = new ProgressBarOptions + var success = await _googlePhotosSvc.LoginAsync(_options); + if (success) { - ProgressCharacter = '─', - ForegroundColor = ConsoleColor.Yellow, - ForegroundColorDone = ConsoleColor.DarkGreen, - BackgroundColor = ConsoleColor.DarkGray, - BackgroundCharacter = '\u2593', - ProgressBarOnBottom = true, - ShowEstimatedDuration = true, - }; - - protected ProgressBarOptions childPbarOptions { get; set; } = new ProgressBarOptions + if (!hasLoggedInBefore && saveDetails) + File.WriteAllText(_optionsFilePath, _options.ToJSON()); + _console.WriteLine("now logged in!! :)"); + } + else + _console.WriteLine("login failed :)"); + } + + protected static void DisplayAlbums(List albums) + { + var headers = new[] { new ColumnHeader("#"), new ColumnHeader("Title"), new ColumnHeader("Items", Alignment.Right), new ColumnHeader("Id") }; + var table = new Table(headers) { Config = TableConfiguration.Markdown() }; + var i = 1; + foreach (var album in albums) { - ProgressCharacter = '─', - ForegroundColor = ConsoleColor.Yellow, - ForegroundColorDone = ConsoleColor.DarkGreen, - BackgroundColor = ConsoleColor.DarkGray, - BackgroundCharacter = '\u2593', - DisplayTimeInRealTime = true, - CollapseWhenFinished = true, - }; + table.AddRow(i, album.title, album.mediaItemsCount, album.id); + i++; + } + Console.Write(table.ToString()); + } + + protected ProgressBarOptions pbarOptions { get; set; } = new ProgressBarOptions + { + ProgressCharacter = '─', + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + BackgroundCharacter = '\u2593', + ProgressBarOnBottom = true, + ShowEstimatedDuration = true, + }; + + protected ProgressBarOptions childPbarOptions { get; set; } = new ProgressBarOptions + { + ProgressCharacter = '─', + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + BackgroundCharacter = '\u2593', + DisplayTimeInRealTime = true, + CollapseWhenFinished = true, + }; - protected ConsoleColor _promptColor = ConsoleColor.White; - protected ConsoleColor _promptBgColor = ConsoleColor.DarkGreen; + protected ConsoleColor _promptColor = ConsoleColor.White; + protected ConsoleColor _promptBgColor = ConsoleColor.DarkGreen; - protected static string _fileDataStoreFullPathOverride => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), AppDomain.CurrentDomain.FriendlyName); + protected static string _fileDataStoreFullPathOverride => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), AppDomain.CurrentDomain.FriendlyName); - protected static string _optionsFilePath => Path.Combine(_fileDataStoreFullPathOverride, $"{nameof(GooglePhotosOptions)}.json"); - protected static string _configFilePath => Path.Combine(_fileDataStoreFullPathOverride, $"{nameof(AppConfig)}.json"); + protected static string _optionsFilePath => Path.Combine(_fileDataStoreFullPathOverride, $"{nameof(GooglePhotosOptions)}.json"); + protected static string _configFilePath => Path.Combine(_fileDataStoreFullPathOverride, $"{nameof(AppConfig)}.json"); - protected GooglePhotosOptions _options; - protected AppConfig _config { get; set; } + protected GooglePhotosOptions _options; + protected AppConfig _config { get; set; } - protected string _userPath; + protected string _userPath; - protected async Task ReadConfig() + protected async Task ReadConfig() + { + if (!File.Exists(_configFilePath)) { - if (!File.Exists(_configFilePath)) - { - _config = new AppConfig(); - await WriteConfig(); - } - else - { - var str = await File.ReadAllTextAsync(_configFilePath); - _config = str.FromJSON(); - } + _config = new AppConfig(); + await WriteConfig(); + } + else + { + var str = await File.ReadAllTextAsync(_configFilePath); + _config = str.FromJSON(); } + } - protected Task WriteConfig() => File.WriteAllTextAsync(_configFilePath, _config.ToJSON()); + protected Task WriteConfig() => File.WriteAllTextAsync(_configFilePath, _config.ToJSON()); - protected async Task SyncMediaItems() - { - allMediaItems = await _diskCacheSvc.GetAsync($"{nameof(allMediaItems)}.json", () => _googlePhotosSvc.GetMediaItemsAsync()); - _config.latestMediaItemCreation = allMediaItems.Max(p => p.mediaMetadata.creationTime); + protected async Task SyncMediaItems() + { + allMediaItems = await _diskCacheSvc.GetAsync($"{nameof(allMediaItems)}.json", () => _googlePhotosSvc.GetMediaItemsAsync()); + _config.latestMediaItemCreation = allMediaItems.Max(p => p.mediaMetadata.creationTime); - //check for duplicate MediaItemIds - it shouldn't be possible to have but somehow I had them... - var duplicateMediaItemIds = allMediaItems.GroupBy(x => x.id).Where(g => g.Count() > 1).Select(y => y.Key).ToList(); - if (!duplicateMediaItemIds.IsNullOrEmpty()) + //check for duplicate MediaItemIds - it shouldn't be possible to have but somehow I had them... + var duplicateMediaItemIds = allMediaItems.GroupBy(x => x.id).Where(g => g.Count() > 1).Select(y => y.Key).ToList(); + if (!duplicateMediaItemIds.IsNullOrEmpty()) + { + _console.WriteLine("Search for and tidy-up the following id duplicates on Google Photos ...this shouldn't be possible but happens!?"); + _console.WriteLine(); + foreach (var id in duplicateMediaItemIds) { - _console.WriteLine("Search for and tidy-up the following id duplicates on Google Photos ...this shouldn't be possible but happens!?"); - _console.WriteLine(); + foreach (var mi in allMediaItems.Where(p => p.id == id)) + _console.WriteLine($"\thttps://photos.google.com/search/{mi.filename}\t{mi.productUrl}"); + } + _console.WriteLine(); + if (Prompt.GetYesNo("Please confirm if you've been able to clean up the above duplicates, i.e. you must delete some!", true, _promptColor, _promptBgColor)) + { + //either we delete the entire cache ad re-get or we manually remove the duplicates + //_diskCacheSvc.Delete("mediaItems.json"); + var records = new List(); foreach (var id in duplicateMediaItemIds) { - foreach (var mi in allMediaItems.Where(p => p.id == id)) - _console.WriteLine($"\thttps://photos.google.com/search/{mi.filename}\t{mi.productUrl}"); - } - _console.WriteLine(); - if (Prompt.GetYesNo("Please confirm if you've been able to clean up the above duplicates, i.e. you must delete some!", true, _promptColor, _promptBgColor)) - { - //either we delete the entire cache ad re-get or we manually remove the duplicates - //_diskCacheSvc.Delete("mediaItems.json"); - var records = new List(); - foreach (var id in duplicateMediaItemIds) - { - var record = allMediaItems.FirstOrDefault(p => p.id == id); - if (record != null) - records.Add(record); - else - throw new Exception($"Cannot find duplicate mediaItem id {id}"); - } - allMediaItems.RemoveAll(p => duplicateMediaItemIds.Contains(p.id)); - allMediaItems.AddRange(records); - File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); - } - else - { - //if we can't delete any, then just remove them before we create a dictionary - //todo: ask Google why duplicates? - allMediaItems.RemoveAll(p => duplicateMediaItemIds.Contains(p.id)); + var record = allMediaItems.FirstOrDefault(p => p.id == id); + if (record != null) + records.Add(record); + else + throw new Exception($"Cannot find duplicate mediaItem id {id}"); } + allMediaItems.RemoveAll(p => duplicateMediaItemIds.Contains(p.id)); + allMediaItems.AddRange(records); + File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); } - dMediaItems = allMediaItems.ToDictionary(k => k.id, v => v); + else + { + //if we can't delete any, then just remove them before we create a dictionary + //todo: ask Google why duplicates? + allMediaItems.RemoveAll(p => duplicateMediaItemIds.Contains(p.id)); + } + } + dMediaItems = allMediaItems.ToDictionary(k => k.id, v => v); - //check for new data every 12 hours - var ts = DateTime.UtcNow.Subtract(_config.lastCheck); - if (ts.TotalHours > 12) + //check for new data every 12 hours + var ts = DateTime.UtcNow.Subtract(_config.lastCheck); + if (ts.TotalHours > 12) + { + //var endDate = DateTime.UtcNow/*.AddDays(1)*/.Date; + //var startDate = mediaItems.Max(p => p.mediaMetadata.creationTime).Date; + //if (startDate < endDate) + + _console.WriteLine($"{ts.TotalHours} hours since last mediaItems sync, now checking for new mediaItems..."); + //get any new photos and add to the mediaItems cache + var newItems = await _googlePhotosSvc.GetMediaItemsByDateRangeAsync(_config.latestMediaItemCreation, DateTime.UtcNow); + if (!newItems.IsNullOrEmpty()) { - //var endDate = DateTime.UtcNow/*.AddDays(1)*/.Date; - //var startDate = mediaItems.Max(p => p.mediaMetadata.creationTime).Date; - //if (startDate < endDate) - - _console.WriteLine($"{ts.TotalHours} hours since last mediaItems sync, now checking for new mediaItems..."); - //get any new photos and add to the mediaItems cache - var newItems = await _googlePhotosSvc.GetMediaItemsByDateRangeAsync(_config.latestMediaItemCreation, DateTime.UtcNow); - if (!newItems.IsNullOrEmpty()) + _console.WriteLine($"{newItems.Count} recent mediaItems discovered..."); + var counter = 0; + foreach (var mi in newItems) { - _console.WriteLine($"{newItems.Count} recent mediaItems discovered..."); - var counter = 0; - foreach (var mi in newItems) + if (dMediaItems.TryAdd(mi.id, mi)) { - if (dMediaItems.TryAdd(mi.id, mi)) - { - allMediaItems.Add(mi); - counter++; - } - else - { - //_logger.LogWarning();//todo: re-enable logging? - Debug.WriteLine(mi); - } + allMediaItems.Add(mi); + counter++; + } + else + { + //_logger.LogWarning();//todo: re-enable logging? + Debug.WriteLine(mi); } - if (counter > 0) - File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); - _console.WriteLine($"added {counter} new mediaItems to local cache"); } + if (counter > 0) + File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); + _console.WriteLine($"added {counter} new mediaItems to local cache"); } - - _config.latestMediaItemCreation = allMediaItems.Max(p => p.mediaMetadata.creationTime); - _config.lastCheck = DateTime.UtcNow; - return true; } - protected async Task SyncAlbums() + _config.latestMediaItemCreation = allMediaItems.Max(p => p.mediaMetadata.creationTime); + _config.lastCheck = DateTime.UtcNow; + return true; + } + + protected async Task SyncAlbums() + { + allAlbums = await _diskCacheSvc.GetAsync($"albums.json", () => _googlePhotosSvc.GetAlbumsAsync()); + foreach (var a in allAlbums) { - allAlbums = await _diskCacheSvc.GetAsync($"albums.json", () => _googlePhotosSvc.GetAlbumsAsync()); - foreach (var a in allAlbums) + //_console.Write(album.title); + var album = await _diskCacheSvc.GetAsync($"album_{a.id}.json", + () => _googlePhotosSvc.GetAlbumAsync(a.id)); + //_console.WriteLine($"\t{alb.mediaItemsCount}"); + + var mediaItemsA = await _diskCacheSvc.GetAsync($"album_mediaItems_{a.id}.json", () => _googlePhotosSvc.GetMediaItemsByAlbumAsync(a.id)); + //todo: we probably need a dictionary check here also... as you never know with this API! + var d = new Dictionary(); + var ids = mediaItemsA.Select(p => p.id).ToList(); + var foundInLookup = dMediaItems.Values.Where(p => ids.Contains(p.id)).ToList(); + if (ids.Count != foundInLookup.Count) { - //_console.Write(album.title); - var album = await _diskCacheSvc.GetAsync($"album_{a.id}.json", - () => _googlePhotosSvc.GetAlbumAsync(a.id)); - //_console.WriteLine($"\t{alb.mediaItemsCount}"); - - var mediaItemsA = await _diskCacheSvc.GetAsync($"album_mediaItems_{a.id}.json", () => _googlePhotosSvc.GetMediaItemsByAlbumAsync(a.id)); - //todo: we probably need a dictionary check here also... as you never know with this API! - var d = new Dictionary(); - var ids = mediaItemsA.Select(p => p.id).ToList(); - var foundInLookup = dMediaItems.Values.Where(p => ids.Contains(p.id)).ToList(); - if (ids.Count != foundInLookup.Count) + foreach (var id in ids) { - foreach (var id in ids) + var mi = GetMI(id); + if (mi is object) + d.TryAdd(id, mi); + else { - var mi = GetMI(id); - if (mi is object) - d.TryAdd(id, mi); - else + //Debugger.Break();//why can't it find the media item!? BURST images don't appear to be returned in the main query? + var obj = await _googlePhotosSvc.GetMediaItemByIdAsync(id);//why is this object not returned in the main query? + if (obj is object) { - //Debugger.Break();//why can't it find the media item!? BURST images don't appear to be returned in the main query? - var obj = await _googlePhotosSvc.GetMediaItemByIdAsync(id);//why is this object not returned in the main query? - if (obj is object) - { - _console.WriteLine(obj.filename); - if (dMediaItems.TryAdd(id, obj))//hmmm do we re-save the main mediaItems cache now? - allMediaItems.Add(obj); - else - throw new Exception("should never get hit"); - mi = obj; - d.TryAdd(id, GetMI(id)); - foundInLookup.Add(mi); - } + _console.WriteLine(obj.filename); + if (dMediaItems.TryAdd(id, obj))//hmmm do we re-save the main mediaItems cache now? + allMediaItems.Add(obj); else - Debugger.Break(); + throw new Exception("should never get hit"); + mi = obj; + d.TryAdd(id, GetMI(id)); + foundInLookup.Add(mi); } - } - if (ids.Count != foundInLookup.Count) - Debugger.Break();//we still have missing images!? - else - { - //todo: re-save cache - File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); - dMediaItems = allMediaItems.ToDictionary(k => k.id, v => v); + else + Debugger.Break(); } } + if (ids.Count != foundInLookup.Count) + Debugger.Break();//we still have missing images!? else - d = allMediaItems.ToDictionary(k => k.id, v => GetMI(v.id)); - - dMediaItemsByAlbum.Add(a.id, d); + { + //todo: re-save cache + File.WriteAllText(Path.Combine(_diskCacheSvc.CacheRoot, "mediaItems.json"), allMediaItems.ToJSON()); + dMediaItems = allMediaItems.ToDictionary(k => k.id, v => v); + } } - dAlbums = allAlbums.ToDictionary(k => k.id, v => v); - return true; + else + d = allMediaItems.ToDictionary(k => k.id, v => GetMI(v.id)); + + dMediaItemsByAlbum.Add(a.id, d); } + dAlbums = allAlbums.ToDictionary(k => k.id, v => v); + return true; + } - /// - /// Performs a search per-category to see how Google has classified the images. - /// - /// - protected async Task SyncMediaItemsByCategory() + /// + /// Performs a search per-category to see how Google has classified the images. + /// + /// + protected async Task SyncMediaItemsByCategory() + { + foreach (var category in Utils.GetAllItems()) { - foreach (var category in Utils.GetAllItems()) + //_console.WriteLine(); + //_console.Write(category); + var mis = await _diskCacheSvc.GetAsync($"mediaItems_{category}.json", + () => _googlePhotosSvc.GetMediaItemsByCategoryAsync(category)); + //_console.WriteLine($"\t{mis.Count}"); + + var d = new Dictionary(); + var duplicateIds = mis.GroupBy(x => x.id).Where(g => g.Count() > 1).Select(y => y.Key).ToList(); + if (duplicateIds.Count > 0) { - //_console.WriteLine(); - //_console.Write(category); - var mis = await _diskCacheSvc.GetAsync($"mediaItems_{category}.json", - () => _googlePhotosSvc.GetMediaItemsByCategoryAsync(category)); - //_console.WriteLine($"\t{mis.Count}"); - - var d = new Dictionary(); - var duplicateIds = mis.GroupBy(x => x.id).Where(g => g.Count() > 1).Select(y => y.Key).ToList(); - if (duplicateIds.Count > 0) - { - //for some reason google returns mutiple mediaItemIds here for a single category... - //Debugger.Break(); - foreach (var item in mis) - d.TryAdd(item.id, item); - } - else - d = mis.ToDictionary(k => k.id, v => GetMI(v.id)); - - dMediaItemsByCategory.Add(category, d); + //for some reason google returns mutiple mediaItemIds here for a single category... + //Debugger.Break(); + foreach (var item in mis) + d.TryAdd(item.id, item); } - return true; - } + else + d = mis.ToDictionary(k => k.id, v => GetMI(v.id)); - protected static List GetAlbumDuplicates(List albums) - { - //check for duplicate album names case-insensitive - //todo: package up the this whole duplicate album by name check? - var duplicateAlbumsByTitle = albums.GroupBy(p => p.title, StringComparer.InvariantCultureIgnoreCase) - .Select(g => new - { - g.Key, - Count = g.Count() - }) - .Where(p => p.Count > 1).ToList(); - return albums.Where(p => duplicateAlbumsByTitle.Select(q => q.Key).Contains(p.title, StringComparer.OrdinalIgnoreCase)).ToList(); + dMediaItemsByCategory.Add(category, d); } + return true; + } - protected List GetFlattened() - { - var lFlattened = new List(allMediaItems.Count); - var dFlattened = new Dictionary(allMediaItems.Count); - foreach (var mediaItem in allMediaItems) + protected static List GetAlbumDuplicates(List albums) + { + //check for duplicate album names case-insensitive + //todo: package up the this whole duplicate album by name check? + var duplicateAlbumsByTitle = albums.GroupBy(p => p.title, StringComparer.InvariantCultureIgnoreCase) + .Select(g => new { - var albumIds = new List(dAlbums.Count); - foreach (var kvp in dMediaItemsByAlbum) - if (kvp.Value.ContainsKey(mediaItem.id)) - albumIds.Add(kvp.Key); + g.Key, + Count = g.Count() + }) + .Where(p => p.Count > 1).ToList(); + return albums.Where(p => duplicateAlbumsByTitle.Select(q => q.Key).Contains(p.title, StringComparer.OrdinalIgnoreCase)).ToList(); + } - var contentCategoryTypes = new List(dMediaItemsByCategory.Count); - foreach (var kvp in dMediaItemsByCategory) - if (kvp.Value.ContainsKey(mediaItem.id)) - contentCategoryTypes.Add(kvp.Key); + protected List GetFlattened() + { + var lFlattened = new List(allMediaItems.Count); + var dFlattened = new Dictionary(allMediaItems.Count); + foreach (var mediaItem in allMediaItems) + { + var albumIds = new List(dAlbums.Count); + foreach (var kvp in dMediaItemsByAlbum) + if (kvp.Value.ContainsKey(mediaItem.id)) + albumIds.Add(kvp.Key); - var o = new flattened - { - id = mediaItem.id, - description = mediaItem.description, - mimeType = mediaItem.mimeType, - filename = mediaItem.filename, - - creationTime = mediaItem.mediaMetadata.creationTime, - width = mediaItem.mediaMetadata.width, - height = mediaItem.mediaMetadata.height, - - focalLength = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.focalLength, - apertureFNumber = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.apertureFNumber, - isoEquivalent = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.isoEquivalent, - exposureTime = mediaItem.isVideo ? null : mediaItem.mediaMetadata.photo.exposureTime, - - fps = mediaItem.isVideo ? mediaItem.mediaMetadata.video.fps : 0, - status = mediaItem.isVideo ? mediaItem.mediaMetadata.video.status : string.Empty, - - cameraMake = mediaItem.isVideo ? mediaItem.mediaMetadata.video.cameraMake : mediaItem.mediaMetadata.photo.cameraMake, - cameraModel = mediaItem.isVideo ? mediaItem.mediaMetadata.video.cameraModel : mediaItem.mediaMetadata.photo.cameraModel, - - albumIds = albumIds.ToArray(), - contentCategoryTypes = contentCategoryTypes.ToArray(), - }; - if (dFlattened.TryAdd(o.id, o)) - lFlattened.Add(o); - else - throw new Exception($"should never get hit?"); - } - return lFlattened; + var contentCategoryTypes = new List(dMediaItemsByCategory.Count); + foreach (var kvp in dMediaItemsByCategory) + if (kvp.Value.ContainsKey(mediaItem.id)) + contentCategoryTypes.Add(kvp.Key); + + var o = new flattened + { + id = mediaItem.id, + description = mediaItem.description, + mimeType = mediaItem.mimeType, + filename = mediaItem.filename, + + creationTime = mediaItem.mediaMetadata.creationTime, + width = mediaItem.mediaMetadata.width, + height = mediaItem.mediaMetadata.height, + + focalLength = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.focalLength, + apertureFNumber = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.apertureFNumber, + isoEquivalent = mediaItem.isVideo ? 0 : mediaItem.mediaMetadata.photo.isoEquivalent, + exposureTime = mediaItem.isVideo ? null : mediaItem.mediaMetadata.photo.exposureTime, + + fps = mediaItem.isVideo ? mediaItem.mediaMetadata.video.fps : 0, + status = mediaItem.isVideo ? mediaItem.mediaMetadata.video.status : string.Empty, + + cameraMake = mediaItem.isVideo ? mediaItem.mediaMetadata.video.cameraMake : mediaItem.mediaMetadata.photo.cameraMake, + cameraModel = mediaItem.isVideo ? mediaItem.mediaMetadata.video.cameraModel : mediaItem.mediaMetadata.photo.cameraModel, + + albumIds = albumIds.ToArray(), + contentCategoryTypes = contentCategoryTypes.ToArray(), + }; + if (dFlattened.TryAdd(o.id, o)) + lFlattened.Add(o); + else + throw new Exception($"should never get hit?"); } + return lFlattened; + } - MediaItem GetMI(string id, [CallerMemberName] string caller = null) + MediaItem GetMI(string id, [CallerMemberName] string caller = null) + { + if (dMediaItems.TryGetValue(id, out var mi)) + return mi; + else { - if (dMediaItems.TryGetValue(id, out var mi)) - return mi; - else - { - Debug.WriteLine($"{nameof(GetMI)} media item not found, caller={caller}"); - return null; - } - //throw new Exception($"possible sync issue? cannot find mi id {id}"); + Debug.WriteLine($"{nameof(GetMI)} media item not found, caller={caller}"); + return null; } + //throw new Exception($"possible sync issue? cannot find mi id {id}"); + } - protected static string GetRelPath(string rootPath, FileInfo fileInfo) => fileInfo.FullName.Replace(rootPath, string.Empty); + protected static string GetRelPath(string rootPath, FileInfo fileInfo) => fileInfo.FullName.Replace(rootPath, string.Empty); - protected static List GetFiles(string path, string searchPattern = "*")//todo: move to Utils in CasCap.Common.Extensions lib + protected static List GetFiles(string path, string searchPattern = "*")//todo: move to Utils in CasCap.Common.Extensions lib + { + var l = new List(); + try { - var l = new List(); - try + if (!Directory.Exists(path)) + return l; + else { - if (!Directory.Exists(path)) - return l; - else + try { - try + foreach (var file in Directory.GetFiles(path, searchPattern)) { - foreach (var file in Directory.GetFiles(path, searchPattern)) + if (File.Exists(file)) { - if (File.Exists(file)) - { - var finfo = new FileInfo(file); - l.Add(finfo); - } + var finfo = new FileInfo(file); + l.Add(finfo); } - foreach (var dir in Directory.GetDirectories(path)) - l.AddRange(GetFiles(dir, searchPattern)); - } - catch (NotSupportedException e) - { - throw new Exception($"Unable to access folder: {e.Message}"); } + foreach (var dir in Directory.GetDirectories(path)) + l.AddRange(GetFiles(dir, searchPattern)); + } + catch (NotSupportedException e) + { + throw new Exception($"Unable to access folder: {e.Message}"); } } - catch (UnauthorizedAccessException e) - { - throw new Exception($"Unable to access folder: {e.Message}"); - } - return l; } + catch (UnauthorizedAccessException e) + { + throw new Exception($"Unable to access folder: {e.Message}"); + } + return l; } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Duplicates.cs b/src/CasCap.GooglePhotosCli/Commands/Duplicates.cs index 1a751c5..f79cf93 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Duplicates.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Duplicates.cs @@ -12,268 +12,268 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command(Description = "Analyse and identify potential duplicate media items in a Google Photos account.")] +internal class Duplicates : CommandBase { - [Command(Description = "Analyse and identify potential duplicate media items in a Google Photos account.")] - internal class Duplicates : CommandBase + public Duplicates(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + + [Argument(0, Description = "Which media type do you wish to analyse?")] + public MediaType type { get; } + + [Option(Description = "Force a cache refresh (this will use up credits!)")] + public bool SkipCache { get; } + + public async override Task OnExecuteAsync(CommandLineApplication app) { - public Duplicates(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + await base.OnExecuteAsync(app); + //await base.OnExecute(cancellationToken); - [Argument(0, Description = "Which media type do you wish to analyse?")] - public MediaType type { get; } + if (!await SyncMediaItems()) return 1; + if (!await SyncAlbums()) return 1; + if (!await SyncMediaItemsByCategory()) return 1; - [Option(Description = "Force a cache refresh (this will use up credits!)")] - public bool SkipCache { get; } + var res = await _diskCacheSvc.GetAsync($"{nameof(ScoreResponse)}_{type}.json", () => GetScoreResponse()); - public async override Task OnExecuteAsync(CommandLineApplication app) + duplicateFolder = Path.Combine(_userPath, "duplicates"); + if (!Directory.Exists(duplicateFolder)) + Directory.CreateDirectory(duplicateFolder); + + //only show where the number of flags set is greater than or equal to minSetFlagsCount + //can't really remember the point in the below investigation atm + /* + var minSetFlagsCount = 3; + var mostCommonScenarios = res.dStats.Where(p => BitOperations.PopCount((ulong)p.Key) >= minSetFlagsCount).OrderByDescending(p => p.Value).ToList(); + foreach (var kvp in mostCommonScenarios) { - await base.OnExecuteAsync(app); - //await base.OnExecute(cancellationToken); + var matchCount = res.dScore.Where(p => p.Value.propertyMatches.HasFlag(kvp.Key)).ToList(); + var str = $"{kvp.Value}\t{kvp.Key}\t{matchCount}"; + _console.WriteLine(str); + } + */ - if (!await SyncMediaItems()) return 1; - if (!await SyncAlbums()) return 1; - if (!await SyncMediaItemsByCategory()) return 1; + _console.WriteLine(); - var res = await _diskCacheSvc.GetAsync($"{nameof(ScoreResponse)}_{type}.json", () => GetScoreResponse()); + var counter = 1; + foreach (var score in res.dScore.OrderByDescending(p => p.Value.count)) + { + var ids = new List(); + if (dMediaItems.TryGetValue(score.Key, out var mi)) + { + ids.Add(mi.id); + var str = $"{score.Value.count}\t{score.Value.propertyMatches}\t"; + var query = new List(); + if (score.Value.propertyMatches.HasFlag(GroupByProperty.filename)) + query.Add(mi.filename); - duplicateFolder = Path.Combine(_userPath, "duplicates"); - if (!Directory.Exists(duplicateFolder)) - Directory.CreateDirectory(duplicateFolder); + if (score.Value.propertyMatches.HasFlag(GroupByProperty.creationTime)) + query.Add(mi.mediaMetadata.creationTime.ToString("yyyy-MM-dd")); - //only show where the number of flags set is greater than or equal to minSetFlagsCount - //can't really remember the point in the below investigation atm - /* - var minSetFlagsCount = 3; - var mostCommonScenarios = res.dStats.Where(p => BitOperations.PopCount((ulong)p.Key) >= minSetFlagsCount).OrderByDescending(p => p.Value).ToList(); - foreach (var kvp in mostCommonScenarios) - { - var matchCount = res.dScore.Where(p => p.Value.propertyMatches.HasFlag(kvp.Key)).ToList(); - var str = $"{kvp.Value}\t{kvp.Key}\t{matchCount}"; + if (query.IsNullOrEmpty()) + str += "??how to identify??"; + else + str += string.Join(", ", query); _console.WriteLine(str); } - */ - - _console.WriteLine(); + else + throw new Exception("should never get hit...?"); - var counter = 1; - foreach (var score in res.dScore.OrderByDescending(p => p.Value.count)) - { - var ids = new List(); - if (dMediaItems.TryGetValue(score.Key, out var mi)) - { - ids.Add(mi.id); - var str = $"{score.Value.count}\t{score.Value.propertyMatches}\t"; - var query = new List(); - if (score.Value.propertyMatches.HasFlag(GroupByProperty.filename)) - query.Add(mi.filename); - - if (score.Value.propertyMatches.HasFlag(GroupByProperty.creationTime)) - query.Add(mi.mediaMetadata.creationTime.ToString("yyyy-MM-dd")); - - if (query.IsNullOrEmpty()) - str += "??how to identify??"; - else - str += string.Join(", ", query); - _console.WriteLine(str); - } - else - throw new Exception("should never get hit...?"); + //get latest versions (i.e. with valid product urls) + var mediaItems = await _googlePhotosSvc.GetMediaItemsByIdsAsync(ids); + await AnalyseExifs(mediaItems); - //get latest versions (i.e. with valid product urls) - var mediaItems = await _googlePhotosSvc.GetMediaItemsByIdsAsync(ids); - await AnalyseExifs(mediaItems); + //https://stackoverflow.com/questions/4580263/how-to-open-in-default-browser-in-c-sharp + //var searchTerm = $"{filename} OR {Path.GetFileNameWithoutExtension(filename)}"; + //var query = System.Web.HttpUtility.HtmlEncode(searchTerm); + //var url = $"https://photos.google.com/search/{query}"; + //if (_options.User.IndexOf("marinos") > -1) + // _console.WriteLine($"{url}"); + //else + //{ + // _console.WriteLine($"{z} of {fileNames.Length}\tHit any key to view potential duplicates for filename '{filename}'..."); + // Console.ReadKey(); + // //Process.Start($"https://photos.google.com/search/{filename}"); + // Process.Start("explorer", url); + //} - //https://stackoverflow.com/questions/4580263/how-to-open-in-default-browser-in-c-sharp - //var searchTerm = $"{filename} OR {Path.GetFileNameWithoutExtension(filename)}"; - //var query = System.Web.HttpUtility.HtmlEncode(searchTerm); - //var url = $"https://photos.google.com/search/{query}"; - //if (_options.User.IndexOf("marinos") > -1) - // _console.WriteLine($"{url}"); - //else - //{ - // _console.WriteLine($"{z} of {fileNames.Length}\tHit any key to view potential duplicates for filename '{filename}'..."); - // Console.ReadKey(); - // //Process.Start($"https://photos.google.com/search/{filename}"); - // Process.Start("explorer", url); - //} + Console.ReadKey(); - Console.ReadKey(); + counter++; + if (counter > 250) break; + } - counter++; - if (counter > 250) break; - } + { + //if (Prompt.GetYesNo("Do you want to clear the local mediaItems cache?", false) + // Utils.DeleteAll(); + //below doesnt work because media items must have been created by the API ... ffs + /* + var albumName = $"{DateTime.UtcNow:yyyy-MM-dd} - duplicates"; + var album = await _googlePhotosSvc.GetOrCreateAlbumAsync(albumName); + if (album is object) { - //if (Prompt.GetYesNo("Do you want to clear the local mediaItems cache?", false) - // Utils.DeleteAll(); - - //below doesnt work because media items must have been created by the API ... ffs - /* - var albumName = $"{DateTime.UtcNow:yyyy-MM-dd} - duplicates"; - var album = await _googlePhotosSvc.GetOrCreateAlbumAsync(albumName); - if (album is object) + _console.WriteLine($"created duplicates album '{albumName}'"); + var ids = duplicateGroupBy.SelectMany(p => p.mediaItems).Select(q => q.id).Distinct().ToArray(); + if (await _googlePhotosSvc.AddMediaItemsToAlbumAsync(album.id, ids)) { - _console.WriteLine($"created duplicates album '{albumName}'"); - var ids = duplicateGroupBy.SelectMany(p => p.mediaItems).Select(q => q.id).Distinct().ToArray(); - if (await _googlePhotosSvc.AddMediaItemsToAlbumAsync(album.id, ids)) - { - _console.WriteLine("added duplicates to album '{albumName}'"); - Process.Start(album.productUrl); - } - else - _console.WriteLine($"unable to add duplicates to album '{albumName}'"); + _console.WriteLine("added duplicates to album '{albumName}'"); + Process.Start(album.productUrl); } else - { - _console.WriteLine($"unable to create duplicates album '{albumName}'"); - return; - } - */ + _console.WriteLine($"unable to add duplicates to album '{albumName}'"); } + else + { + _console.WriteLine($"unable to create duplicates album '{albumName}'"); + return; + } + */ + } - //compare post-download; - // file size (can't get this) - // exif tags (which tags are important and which can we disregard?) + //compare post-download; + // file size (can't get this) + // exif tags (which tags are important and which can we disregard?) - //todo: record all query history, either in a log, or a summary json file to show if we are approaching the daily limit? - //todo: download/cache all items per album - //todo: upload (nested) directory structure? w/console ui progress indicator? w/webp conversion? - //todo: download all media items to a local cache? - //todo: re-order all media items based on creationTime? (the default album order is when added) - //todo: add Console.Clear to McMaster.Extensions.CommandLineUtils? + //todo: record all query history, either in a log, or a summary json file to show if we are approaching the daily limit? + //todo: download/cache all items per album + //todo: upload (nested) directory structure? w/console ui progress indicator? w/webp conversion? + //todo: download all media items to a local cache? + //todo: re-order all media items based on creationTime? (the default album order is when added) + //todo: add Console.Clear to McMaster.Extensions.CommandLineUtils? - await WriteConfig(); - return 0; - } + await WriteConfig(); + return 0; + } - static long iteration = 1; + static long iteration = 1; - async Task GetScoreResponse() - { - await Task.Delay(0); + async Task GetScoreResponse() + { + await Task.Delay(0); - var res = new ScoreResponse(); + var res = new ScoreResponse(); - var flattened = GetFlattened(); + var flattened = GetFlattened(); - //todo: for speed analyse flattened to see if Count(DISTINCT mimeType) > 1 - if mimeType only ever null OR jpg, then don't include in combinations (repeat for all other fields) - //todo: split photos and video duplication check + //todo: for speed analyse flattened to see if Count(DISTINCT mimeType) > 1 - if mimeType only ever null OR jpg, then don't include in combinations (repeat for all other fields) + //todo: split photos and video duplication check - var lGroupByCombinations = Utils.GetAllCombinations().ToList(); + var lGroupByCombinations = Utils.GetAllCombinations().ToList(); - if (type == MediaType.Photo) - { - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.fps)); - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.status)); - flattened = flattened.Where(p => p.mimeType.StartsWith("image", StringComparison.OrdinalIgnoreCase)).ToList(); - } - if (type == MediaType.Video) - { - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.focalLength)); - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.apertureFNumber)); - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.isoEquivalent)); - lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.exposureTime)); - flattened = flattened.Where(p => p.mimeType.StartsWith("video", StringComparison.OrdinalIgnoreCase)).ToList(); - } + if (type == MediaType.Photo) + { + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.fps)); + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.status)); + flattened = flattened.Where(p => p.mimeType.StartsWith("image", StringComparison.OrdinalIgnoreCase)).ToList(); + } + if (type == MediaType.Video) + { + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.focalLength)); + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.apertureFNumber)); + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.isoEquivalent)); + lGroupByCombinations.RemoveAll(p => p.HasFlag(GroupByProperty.exposureTime)); + flattened = flattened.Where(p => p.mimeType.StartsWith("video", StringComparison.OrdinalIgnoreCase)).ToList(); + } - //flattened = flattened.Take(2_000).ToList();//make testing faster! + //flattened = flattened.Take(2_000).ToList();//make testing faster! - var msg = $"Now processing {flattened.Count} records against {lGroupByCombinations.Count} property combinations."; + var msg = $"Now processing {flattened.Count} records against {lGroupByCombinations.Count} property combinations."; - var dtStart = DateTime.UtcNow; - var estimatedDuration = TimeSpan.FromMilliseconds(lGroupByCombinations.Count * 25); - if (false) + var dtStart = DateTime.UtcNow; + var estimatedDuration = TimeSpan.FromMilliseconds(lGroupByCombinations.Count * 25); + if (false) + { + using var pbar = new ProgressBar(lGroupByCombinations.Count, msg, pbarOptions) { - using var pbar = new ProgressBar(lGroupByCombinations.Count, msg, pbarOptions) - { - EstimatedDuration = estimatedDuration - }; - foreach (var myGroup in lGroupByCombinations) - { - CalculateScore(myGroup); - UpdateProgress(pbar, myGroup); - } - } - else + EstimatedDuration = estimatedDuration + }; + foreach (var myGroup in lGroupByCombinations) { - var cts = new CancellationTokenSource(); + CalculateScore(myGroup); + UpdateProgress(pbar, myGroup); + } + } + else + { + var cts = new CancellationTokenSource(); - var po = new ParallelOptions { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; + var po = new ParallelOptions { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; - // Run a task so that we can cancel from another thread. + // Run a task so that we can cancel from another thread. #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - Task.Factory.StartNew(() => - { - if (Console.ReadKey().KeyChar == 'c') - cts.Cancel(); - Console.WriteLine("press any key to exit"); - }); + Task.Factory.StartNew(() => + { + if (Console.ReadKey().KeyChar == 'c') + cts.Cancel(); + Console.WriteLine("press any key to exit"); + }); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - try + try + { + using var pbar = new ProgressBar(lGroupByCombinations.Count, msg, pbarOptions) + { + EstimatedDuration = estimatedDuration + }; + Parallel.ForEach(lGroupByCombinations, po, (myGroup) => { - using var pbar = new ProgressBar(lGroupByCombinations.Count, msg, pbarOptions) - { - EstimatedDuration = estimatedDuration - }; - Parallel.ForEach(lGroupByCombinations, po, (myGroup) => - { //using (var child = pbar.Spawn(1, myGroup.ToString(), childOptions)) //{ CalculateScore(myGroup); - UpdateProgress(pbar, myGroup); + UpdateProgress(pbar, myGroup); // child.Tick(); //todo: can't exit early now with progress bar? po.CancellationToken.ThrowIfCancellationRequested(); //} //pbar.Tick(); }); - } - catch (OperationCanceledException e) - { - Console.WriteLine(e.Message); - } - finally - { - cts.Dispose(); - } } - return res; - - void CalculateScore(GroupByProperty myGroup) + catch (OperationCanceledException e) { - Interlocked.Increment(ref iteration); - //https://stackoverflow.com/questions/33286297/linq-groupby-with-a-dynamic-group-of-columns - var results = flattened - .GroupBy(item => - new - { - filename = myGroup.HasFlag(GroupByProperty.filename) ? item.filename : null, - mimeType = myGroup.HasFlag(GroupByProperty.mimeType) ? item.mimeType : null, - dimensions = myGroup.HasFlag(GroupByProperty.dimensions) ? $"{item.height}x{item.width}" : null, - creationTime = myGroup.HasFlag(GroupByProperty.creationTime) ? item.creationTime : DateTime.MinValue, - description = myGroup.HasFlag(GroupByProperty.description) ? item.description : null, + Console.WriteLine(e.Message); + } + finally + { + cts.Dispose(); + } + } + return res; + + void CalculateScore(GroupByProperty myGroup) + { + Interlocked.Increment(ref iteration); + //https://stackoverflow.com/questions/33286297/linq-groupby-with-a-dynamic-group-of-columns + var results = flattened + .GroupBy(item => + new + { + filename = myGroup.HasFlag(GroupByProperty.filename) ? item.filename : null, + mimeType = myGroup.HasFlag(GroupByProperty.mimeType) ? item.mimeType : null, + dimensions = myGroup.HasFlag(GroupByProperty.dimensions) ? $"{item.height}x{item.width}" : null, + creationTime = myGroup.HasFlag(GroupByProperty.creationTime) ? item.creationTime : DateTime.MinValue, + description = myGroup.HasFlag(GroupByProperty.description) ? item.description : null, //photo focalLength = myGroup.HasFlag(GroupByProperty.focalLength) ? item.focalLength : float.MinValue, - apertureFNumber = myGroup.HasFlag(GroupByProperty.apertureFNumber) ? item.apertureFNumber : float.MinValue, - isoEquivalent = myGroup.HasFlag(GroupByProperty.isoEquivalent) ? item.isoEquivalent : int.MinValue, - exposureTime = myGroup.HasFlag(GroupByProperty.exposureTime) ? item.exposureTime : null, + apertureFNumber = myGroup.HasFlag(GroupByProperty.apertureFNumber) ? item.apertureFNumber : float.MinValue, + isoEquivalent = myGroup.HasFlag(GroupByProperty.isoEquivalent) ? item.isoEquivalent : int.MinValue, + exposureTime = myGroup.HasFlag(GroupByProperty.exposureTime) ? item.exposureTime : null, //video fps = myGroup.HasFlag(GroupByProperty.fps) ? item.fps : float.MinValue, - status = myGroup.HasFlag(GroupByProperty.status) ? item.status : null, + status = myGroup.HasFlag(GroupByProperty.status) ? item.status : null, //photo & video cameraMake = myGroup.HasFlag(GroupByProperty.cameraMake) ? item.cameraMake : null, - cameraModel = myGroup.HasFlag(GroupByProperty.cameraModel) ? item.cameraModel : null, + cameraModel = myGroup.HasFlag(GroupByProperty.cameraModel) ? item.cameraModel : null, //collections albumIds = myGroup.HasFlag(GroupByProperty.albumIds) ? item.albumIds : null, - contentCategoryTypes = myGroup.HasFlag(GroupByProperty.contentCategoryTypes) ? item.contentCategoryTypes : null, - }) - .Select(g => new - { + contentCategoryTypes = myGroup.HasFlag(GroupByProperty.contentCategoryTypes) ? item.contentCategoryTypes : null, + }) + .Select(g => new + { //g.Key.filename, //g.Key.mimeType, //g.Key.dimensions, @@ -295,105 +295,104 @@ void CalculateScore(GroupByProperty myGroup) //g.Key.contentCategoryTypes, //quantity = i.Sum() medaItemIds = g.Select(p => p.id).ToList(), - count = g.Count() - }).Where(p => p.count > 1).AsParallel().ToList(); - - //res.dScore2.AddOrUpdate(myGroup, results.medaItemIds, (k, v) => v = result.medaItemIds); - - //Parallel.ForEach ?? - foreach (var result in results) - {//todo: we can also record on what properties/enums the matches occurred (those properties could appear in an enum checkbox list?) - foreach (var id in result.medaItemIds) - _ = res.dScore.AddOrUpdate(id, new MediaItemScore { count = 1, propertyMatches = myGroup }, (k, v) => - { - v.count++; - v.propertyMatches |= myGroup; - return v; - }); - //record a count of matches per bitmask - _ = res.dStats.AddOrUpdate(myGroup, result.medaItemIds.Count, (k, v) => + count = g.Count() + }).Where(p => p.count > 1).AsParallel().ToList(); + + //res.dScore2.AddOrUpdate(myGroup, results.medaItemIds, (k, v) => v = result.medaItemIds); + + //Parallel.ForEach ?? + foreach (var result in results) + {//todo: we can also record on what properties/enums the matches occurred (those properties could appear in an enum checkbox list?) + foreach (var id in result.medaItemIds) + _ = res.dScore.AddOrUpdate(id, new MediaItemScore { count = 1, propertyMatches = myGroup }, (k, v) => { - v = result.medaItemIds.Count; + v.count++; + v.propertyMatches |= myGroup; return v; }); - //_ = res.dScore2.AddOrUpdate(myGroup, new HashSet(result.medaItemIds), (k, v) => - //{ - // //v.AddRange(result.medaItemIds); - // foreach (var _id in result.medaItemIds) - // { - // if (!v.Contains(_id)) - // v.Add(_id); - // else - // Debugger.Break(); - // } - // return v; - //}); - } + //record a count of matches per bitmask + _ = res.dStats.AddOrUpdate(myGroup, result.medaItemIds.Count, (k, v) => + { + v = result.medaItemIds.Count; + return v; + }); + //_ = res.dScore2.AddOrUpdate(myGroup, new HashSet(result.medaItemIds), (k, v) => + //{ + // //v.AddRange(result.medaItemIds); + // foreach (var _id in result.medaItemIds) + // { + // if (!v.Contains(_id)) + // v.Add(_id); + // else + // Debugger.Break(); + // } + // return v; + //}); } + } - void UpdateProgress(ProgressBar pbar, GroupByProperty myGroup) + void UpdateProgress(ProgressBar pbar, GroupByProperty myGroup) + { + //pbar.Tick(); + pbar.Tick($"Iteration {iteration} of {lGroupByCombinations.Count} completed, {myGroup}"); + if (Interlocked.Read(ref iteration) % 25 == 0) { - //pbar.Tick(); - pbar.Tick($"Iteration {iteration} of {lGroupByCombinations.Count} completed, {myGroup}"); - if (Interlocked.Read(ref iteration) % 25 == 0) - { - var tsTaken = DateTime.UtcNow.Subtract(dtStart).TotalMilliseconds; - var timePerCombination = tsTaken / iteration; - pbar.EstimatedDuration = TimeSpan.FromMilliseconds((lGroupByCombinations.Count - iteration) * timePerCombination); - } + var tsTaken = DateTime.UtcNow.Subtract(dtStart).TotalMilliseconds; + var timePerCombination = tsTaken / iteration; + pbar.EstimatedDuration = TimeSpan.FromMilliseconds((lGroupByCombinations.Count - iteration) * timePerCombination); } } + } - async Task AnalyseExifs(List mediaItems) + async Task AnalyseExifs(List mediaItems) + { + _console.WriteLine($"Download byte data;"); + var j = 1; + foreach (var mediaItem in mediaItems) { - _console.WriteLine($"Download byte data;"); - var j = 1; - foreach (var mediaItem in mediaItems) - { - _console.WriteLine($"{j})\turl => {mediaItem.productUrl}"); + _console.WriteLine($"{j})\turl => {mediaItem.productUrl}"); - var filePath = Path.Combine(duplicateFolder, mediaItem.id); - if (!File.Exists(filePath)) - { - _console.Write($"{j})\tdownloading {mediaItem.id} ..."); - var bytes = await _googlePhotosSvc.DownloadBytes(mediaItem, download: true); - await File.WriteAllBytesAsync(filePath, bytes); - _console.WriteLine($"downloaded {((long)bytes.Length).GetSizeInMB()}MB");//todo: need a KB extension method here? - } - else - _console.WriteLine($"{j})\timage already exists {mediaItem.id}"); - j++; + var filePath = Path.Combine(duplicateFolder, mediaItem.id); + if (!File.Exists(filePath)) + { + _console.Write($"{j})\tdownloading {mediaItem.id} ..."); + var bytes = await _googlePhotosSvc.DownloadBytes(mediaItem, download: true); + await File.WriteAllBytesAsync(filePath, bytes); + _console.WriteLine($"downloaded {((long)bytes.Length).GetSizeInMB()}MB");//todo: need a KB extension method here? } + else + _console.WriteLine($"{j})\timage already exists {mediaItem.id}"); + j++; + } - _console.WriteLine(); - _console.WriteLine($"Extract EXIF data;"); - j = 1; + _console.WriteLine(); + _console.WriteLine($"Extract EXIF data;"); + j = 1; - var lTags = new List<(string mediaItemId, ExifTag tag, ExifDataType tagDataType, object tagValue)>(); - foreach (var mediaItem in mediaItems) - { - var filePath = Path.Combine(duplicateFolder, mediaItem.id); + var lTags = new List<(string mediaItemId, ExifTag tag, ExifDataType tagDataType, object tagValue)>(); + foreach (var mediaItem in mediaItems) + { + var filePath = Path.Combine(duplicateFolder, mediaItem.id); - _console.WriteLine($"{j})\t{filePath}"); - using (var image = Image.Load(filePath)) + _console.WriteLine($"{j})\t{filePath}"); + using (var image = Image.Load(filePath)) + { + foreach (var tag in image.Metadata.ExifProfile.Values) { - foreach (var tag in image.Metadata.ExifProfile.Values) - { - lTags.Add((mediaItem.id, tag.Tag, tag.DataType, tag.GetValue())); - //_console.WriteLine($"{tag.DataType}\t{tag.Tag}\t={tag.GetValue()}"); - } + lTags.Add((mediaItem.id, tag.Tag, tag.DataType, tag.GetValue())); + //_console.WriteLine($"{tag.DataType}\t{tag.Tag}\t={tag.GetValue()}"); } - j++; } + j++; + } - _console.WriteLine(); - _console.WriteLine($"Summarise/group EXIF data;"); - var uniqueTags = lTags.Select(p => p.tag).Distinct(); - foreach (var tag in uniqueTags) - { - var values = lTags.Where(p => p.tag == tag).Select(p => p.tagValue.ToString()).Distinct().ToList(); - _console.WriteLine($"{tag} ({values.Count})\t=\t{string.Join(", ", values)}"); - } + _console.WriteLine(); + _console.WriteLine($"Summarise/group EXIF data;"); + var uniqueTags = lTags.Select(p => p.tag).Distinct(); + foreach (var tag in uniqueTags) + { + var values = lTags.Where(p => p.tag == tag).Select(p => p.tagValue.ToString()).Distinct().ToList(); + _console.WriteLine($"{tag} ({values.Count})\t=\t{string.Join(", ", values)}"); } } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Logout.cs b/src/CasCap.GooglePhotosCli/Commands/Logout.cs index 72f52ec..530482b 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Logout.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Logout.cs @@ -1,20 +1,18 @@ using CasCap.Services; using McMaster.Extensions.CommandLineUtils; -using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command(Description = "Sign-out and delete all local data.")] +internal class Logout : CommandBase { - [Command(Description = "Sign-out and delete all local data.")] - internal class Logout : CommandBase - { - public Logout(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + public Logout(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); - _console.WriteLine($"todo: need to implement this..."); + _console.WriteLine($"todo: need to implement this..."); - return 0; - } + return 0; } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Mediaitems.cs b/src/CasCap.GooglePhotosCli/Commands/Mediaitems.cs index d3239f0..ee9e689 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Mediaitems.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Mediaitems.cs @@ -3,46 +3,45 @@ using CasCap.Services; using McMaster.Extensions.CommandLineUtils; using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command("mediaitems", Description = "Manage your media items i.e. photos & videos")] +[Subcommand(typeof(Upload))] +[Subcommand(typeof(List))] +[Subcommand(typeof(Duplicates))] +internal class MediaItems : CommandBase { - [Command("mediaitems", Description = "Manage your media items i.e. photos & videos")] - [Subcommand(typeof(Upload))] - [Subcommand(typeof(List))] - [Subcommand(typeof(Duplicates))] - internal class MediaItems : CommandBase + public MediaItems(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); + _console.Error.WriteLine("You must specify an action. See --help for more details."); + return 1; + } + + [Command(Description = "List media items")] + class List : CommandBase { - public MediaItems(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + public List(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } public async override Task OnExecuteAsync(CommandLineApplication app) { await base.OnExecuteAsync(app); - _console.Error.WriteLine("You must specify an action. See --help for more details."); - return 1; - } - - [Command(Description = "List media items")] - class List : CommandBase - { - public List(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - - public async override Task OnExecuteAsync(CommandLineApplication app) + var mediaitems = await _googlePhotosSvc.GetMediaItemsAsync(); + if (mediaitems.IsNullOrEmpty()) { - await base.OnExecuteAsync(app); - var mediaitems = await _googlePhotosSvc.GetMediaItemsAsync(); - if (mediaitems.IsNullOrEmpty()) - { - _console.WriteLine("Sorry, no media items available..."); - return 0; - } - var table = new Table(string.Empty, "File Name", "Mime Type", "Id") { Config = TableConfiguration.Markdown() }; - var i = 1; - foreach (var mediaitem in mediaitems) - { - table.AddRow(i, mediaitem.filename, mediaitem.mimeType, mediaitem.id); - i++; - } + _console.WriteLine("Sorry, no media items available..."); return 0; } + var table = new Table(string.Empty, "File Name", "Mime Type", "Id") { Config = TableConfiguration.Markdown() }; + var i = 1; + foreach (var mediaitem in mediaitems) + { + table.AddRow(i, mediaitem.filename, mediaitem.mimeType, mediaitem.id); + i++; + } + return 0; } } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Sync.cs b/src/CasCap.GooglePhotosCli/Commands/Sync.cs index 9dbcfe1..827c23f 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Sync.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Sync.cs @@ -1,22 +1,20 @@ using CasCap.Services; using McMaster.Extensions.CommandLineUtils; -using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command(Description = "Synchronise media item and album data from remote to local.")] +internal class Sync : CommandBase { - [Command(Description = "Synchronise media item and album data from remote to local.")] - internal class Sync : CommandBase - { - public Sync(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } + public Sync(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); - if (!await SyncMediaItems()) return 1; - if (!await SyncAlbums()) return 1; - if (!await SyncMediaItemsByCategory()) return 1; + if (!await SyncMediaItems()) return 1; + if (!await SyncAlbums()) return 1; + if (!await SyncMediaItemsByCategory()) return 1; - return 0; - } + return 0; } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Commands/Upload.cs b/src/CasCap.GooglePhotosCli/Commands/Upload.cs index 56f32e7..f0ba09f 100644 --- a/src/CasCap.GooglePhotosCli/Commands/Upload.cs +++ b/src/CasCap.GooglePhotosCli/Commands/Upload.cs @@ -5,287 +5,281 @@ using McMaster.Extensions.CommandLineUtils; using MimeTypes; using ShellProgressBar; -using System; -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -namespace CasCap.Commands +namespace CasCap.Commands; + +[Command(Description = "Upload media items to Google Photos account.")] +internal class Upload : CommandBase { - [Command(Description = "Upload media items to Google Photos account.")] - internal class Upload : CommandBase + public Upload(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) { - public Upload(IConsole console, DiskCacheService diskCacheSvc, GooglePhotosService googlePhotosSvc) : base(console, diskCacheSvc, googlePhotosSvc) - { - _googlePhotosSvc.UploadProgressEvent += _googlePhotosSvc_UploadProgressEvent; - } + _googlePhotosSvc.UploadProgressEvent += _googlePhotosSvc_UploadProgressEvent; + } - [Required] - [Option("-s|--source", Description = "Path to media item or folder root.")] - public string path { get; } + [Required] + [Option("-s|--source", Description = "Path to media item or folder root.")] + public string path { get; } - [Option("--pattern", Description = "Inclusive folder wildcard filter (defaults to all google supported extensions).")] - public string searchPattern { get; } = "*.*";//autodetect + [Option("--pattern", Description = "Inclusive folder wildcard filter (defaults to all google supported extensions).")] + public string searchPattern { get; } = "*.*";//autodetect - [Option("--webp", Description = "Convert and upload media items in WEBP format.")] - public bool webp { get; }//how to set webp value? + [Option("--webp", Description = "Convert and upload media items in WEBP format.")] + public bool webp { get; }//how to set webp value? - [Option("-d|--delete", Description = "Delete local media album after successful upload.")] - public bool deleteLocal { get; } + [Option("-d|--delete", Description = "Delete local media album after successful upload.")] + public bool deleteLocal { get; } - [Option("-t|--title", Description = "Upload to album with this title.")] - public string albumTitle { get; } + [Option("-t|--title", Description = "Upload to album with this title.")] + public string albumTitle { get; } - [Option("-h|--hierarchy", Description = "Upload to albums based on folder names.")] - public bool albumHierarchy { get; } + [Option("-h|--hierarchy", Description = "Upload to albums based on folder names.")] + public bool albumHierarchy { get; } - [Option("-y|--yes", Description = "Assume Yes.")] - public bool AutoConfirm { get; } + [Option("-y|--yes", Description = "Assume Yes.")] + public bool AutoConfirm { get; } - //create album if not found? + //create album if not found? - void _googlePhotosSvc_UploadProgressEvent(object sender, UploadProgressArgs e) - { - var str = $"{e.fileName} : {(int)e.uploadedBytes.GetSizeInKB()} of {(int)e.totalBytes.GetSizeInKB()} Kb"; - Debug.WriteLine(str); - childPBar.Tick((int)e.uploadedBytes, str); - } + void _googlePhotosSvc_UploadProgressEvent(object sender, UploadProgressArgs e) + { + var str = $"{e.fileName} : {(int)e.uploadedBytes.GetSizeInKB()} of {(int)e.totalBytes.GetSizeInKB()} Kb"; + Debug.WriteLine(str); + childPBar.Tick((int)e.uploadedBytes, str); + } - public async override Task OnExecuteAsync(CommandLineApplication app) - { - await base.OnExecuteAsync(app); + public async override Task OnExecuteAsync(CommandLineApplication app) + { + await base.OnExecuteAsync(app); - var rootPath = Path.GetFullPath(path); + var rootPath = Path.GetFullPath(path); - _console.Write($"Checking for file(s)... "); - var allFileInfos = GetFiles(path, searchPattern); - var items = new List(allFileInfos.Count); + _console.Write($"Checking for file(s)... "); + var allFileInfos = GetFiles(path, searchPattern); + var items = new List(allFileInfos.Count); - if (allFileInfos.IsNullOrEmpty()) - _console.WriteLine($" 0 files found at {rootPath}"); - else + if (allFileInfos.IsNullOrEmpty()) + _console.WriteLine($" 0 files found at {rootPath}"); + else + { + var checkForUploadableFileTypes = allFileInfos.GroupBy(p => Path.GetExtension(p.Name), StringComparer.InvariantCultureIgnoreCase) + .Select(g => new + { + Extension = g.Key, + MimeType = MimeTypeMap.GetMimeType(g.Key), + Count = g.Count(), + TotalBytes = g.Sum(p => p.Length) + }) + .ToList(); + _console.WriteLine($"located {allFileInfos.Count} file(s), breakdown of file types;"); + _console.WriteLine(); + //todo: do we also analyse the files with ImageSharp/Exif? + + var headers = new[] { new ColumnHeader("File Extension"), new ColumnHeader("Mime Type"), new ColumnHeader("Count", Alignment.Right), new ColumnHeader("Size (MB)", Alignment.Right), new ColumnHeader("Status") }; + var table = new Table(headers) { Config = TableConfiguration.Markdown() }; + foreach (var f in checkForUploadableFileTypes.OrderBy(p => p.Extension)) { - var checkForUploadableFileTypes = allFileInfos.GroupBy(p => Path.GetExtension(p.Name), StringComparer.InvariantCultureIgnoreCase) - .Select(g => new - { - Extension = g.Key, - MimeType = MimeTypeMap.GetMimeType(g.Key), - Count = g.Count(), - TotalBytes = g.Sum(p => p.Length) - }) - .ToList(); - _console.WriteLine($"located {allFileInfos.Count} file(s), breakdown of file types;"); - _console.WriteLine(); - //todo: do we also analyse the files with ImageSharp/Exif? + var status = string.Empty; + if (!GooglePhotosService.IsFileUploadableByExtension(f.Extension)) + status = "Unsupported file extension, will not be uploaded."; + table.AddRow(f.Extension, f.MimeType, f.Count, f.TotalBytes.GetSizeInMB().ToString("0.0"), status); + } + //below summary row breaks the progress bar somehow + //table.AddRow(string.Empty, string.Empty, allFileInfos.Count, allFileInfos.Sum(p => p.Length.GetSizeInMB()).ToString("0.0"), string.Empty); + Console.Write(table.ToString()); + _console.WriteLine(); + + //add all uploadable files into a new collection + foreach (var fileInfo in allFileInfos) + if (GooglePhotosService.IsFileUploadable(fileInfo.FullName)) + items.Add(new MyMediaFileItem { fileInfo = fileInfo }); + } + if (items.IsNullOrEmpty()) + { + _console.WriteLine($"{items.Count} uploadable file(s)"); + return 0; + } - var headers = new[] { new ColumnHeader("File Extension"), new ColumnHeader("Mime Type"), new ColumnHeader("Count", Alignment.Right), new ColumnHeader("Size (MB)", Alignment.Right), new ColumnHeader("Status") }; - var table = new Table(headers) { Config = TableConfiguration.Markdown() }; - foreach (var f in checkForUploadableFileTypes.OrderBy(p => p.Extension)) - { - var status = string.Empty; - if (!GooglePhotosService.IsFileUploadableByExtension(f.Extension)) - status = "Unsupported file extension, will not be uploaded."; - table.AddRow(f.Extension, f.MimeType, f.Count, f.TotalBytes.GetSizeInMB().ToString("0.0"), status); - } - //below summary row breaks the progress bar somehow - //table.AddRow(string.Empty, string.Empty, allFileInfos.Count, allFileInfos.Sum(p => p.Length.GetSizeInMB()).ToString("0.0"), string.Empty); - Console.Write(table.ToString()); - _console.WriteLine(); + { + //extract album information from the folder structure (if requested) + _console.WriteLine($"{items.Count} file(s) to be uploaded;"); + _console.WriteLine(); - //add all uploadable files into a new collection - foreach (var fileInfo in allFileInfos) - if (GooglePhotosService.IsFileUploadable(fileInfo.FullName)) - items.Add(new MyMediaFileItem { fileInfo = fileInfo }); - } - if (items.IsNullOrEmpty()) + var headers = new[] { new ColumnHeader("Relative Path"), new ColumnHeader("Size (KB)", Alignment.Right), new ColumnHeader("Album(s)") }; + var table = new Table(headers) { Config = TableConfiguration.Markdown() }; + foreach (var item in items) { - _console.WriteLine($"{items.Count} uploadable file(s)"); - return 0; + item.relPath = GetRelPath(rootPath, item.fileInfo); + item.albums = GetAlbums(item); + table.AddRow(item.relPath, item.fileInfo.Length.GetSizeInKB().ToString("#,###,###"), string.Join(", ", item.albums)); } + Console.Write(table.ToString()); + _console.WriteLine(); + } - { - //extract album information from the folder structure (if requested) - _console.WriteLine($"{items.Count} file(s) to be uploaded;"); - _console.WriteLine(); - var headers = new[] { new ColumnHeader("Relative Path"), new ColumnHeader("Size (KB)", Alignment.Right), new ColumnHeader("Album(s)") }; - var table = new Table(headers) { Config = TableConfiguration.Markdown() }; - foreach (var item in items) - { - item.relPath = GetRelPath(rootPath, item.fileInfo); - item.albums = GetAlbums(item); - table.AddRow(item.relPath, item.fileInfo.Length.GetSizeInKB().ToString("#,###,###"), string.Join(", ", item.albums)); - } - Console.Write(table.ToString()); - _console.WriteLine(); - } + string[] GetAlbums(MyMediaFileItem item) + { + var albums = item.relPath.Substring(0, item.relPath.LastIndexOf(item.fileInfo.Name)); + if (albums.StartsWith(Path.DirectorySeparatorChar)) albums = albums.Substring(1); + if (albums.EndsWith(Path.DirectorySeparatorChar)) albums = albums.Substring(0, albums.Length - 1); + var myAlbums = albums.Split(Path.DirectorySeparatorChar); + return myAlbums; + } - string[] GetAlbums(MyMediaFileItem item) - { - var albums = item.relPath.Substring(0, item.relPath.LastIndexOf(item.fileInfo.Name)); - if (albums.StartsWith(Path.DirectorySeparatorChar)) albums = albums.Substring(1); - if (albums.EndsWith(Path.DirectorySeparatorChar)) albums = albums.Substring(0, albums.Length - 1); - var myAlbums = albums.Split(Path.DirectorySeparatorChar); - return myAlbums; - } + //note: if we are uploading a crazy amount of data ProgressBar only supports int for ticks, so may break :/ + var totalBytes = items.Sum(p => p.fileInfo.Length); + if (totalBytes > int.MaxValue) + throw new Exception($"Unable to upload more than {((long)int.MaxValue).GetSizeInMB()} in one session!"); + var totalKBytes = totalBytes.GetSizeInKB(); - //note: if we are uploading a crazy amount of data ProgressBar only supports int for ticks, so may break :/ - var totalBytes = items.Sum(p => p.fileInfo.Length); - if (totalBytes > int.MaxValue) - throw new Exception($"Unable to upload more than {((long)int.MaxValue).GetSizeInMB()} in one session!"); + if (!AutoConfirm && !Prompt.GetYesNo($"Hit (Y)es to upload {items.Count} files, {totalBytes.GetSizeInMB():###,###} MB...", false, ConsoleColor.Cyan)) + return 0; + else + _console.WriteLine($"Now uploading {items.Count} files, {totalBytes.GetSizeInMB():###,###} MB..."); - var totalKBytes = totalBytes.GetSizeInKB(); + var dtStart = DateTime.UtcNow; + var estimatedDuration = TimeSpan.FromMilliseconds(items.Count * 2_000);//set gu-estimatedDuration - if (!AutoConfirm && !Prompt.GetYesNo($"Hit (Y)es to upload {items.Count} files, {totalBytes.GetSizeInMB():###,###} MB...", false, ConsoleColor.Cyan)) - return 0; + pbar = new ProgressBar((int)totalBytes, $"Uploading {items.Count} media item(s)...", pbarOptions) + { + EstimatedDuration = estimatedDuration + }; + + //do we upload an assign to library(and albums) as we progress? + //or + //do we upload all and get the uploadTokens, then assign to the library(and albums) in a second step? + //...which gives the user to bomb out if a file isn't successfully uploaded? + var uploadedFileCount = 0; + var uploadedTotalBytes = 0; + foreach (var item in items) + { + var str = $"{item.fileInfo.Name} : 0 of {(int)item.fileInfo.Length.GetSizeInKB()} Kb"; + childPBar = pbar.Spawn((int)item.fileInfo.Length, str, childPbarOptions); + + //todo: pass Action or Func for a callback instead of raising an event? + var uploadToken = await _googlePhotosSvc.UploadMediaAsync(item.fileInfo.FullName/*, callback: child.Tick()*/); + if (!string.IsNullOrWhiteSpace(uploadToken)) + item.uploadToken = uploadToken; else - _console.WriteLine($"Now uploading {items.Count} files, {totalBytes.GetSizeInMB():###,###} MB..."); + { + Debugger.Break(); + //todo: how to handle upload failure here? + } - var dtStart = DateTime.UtcNow; - var estimatedDuration = TimeSpan.FromMilliseconds(items.Count * 2_000);//set gu-estimatedDuration + childPBar.Dispose(); - pbar = new ProgressBar((int)totalBytes, $"Uploading {items.Count} media item(s)...", pbarOptions) - { - EstimatedDuration = estimatedDuration - }; - - //do we upload an assign to library(and albums) as we progress? - //or - //do we upload all and get the uploadTokens, then assign to the library(and albums) in a second step? - //...which gives the user to bomb out if a file isn't successfully uploaded? - var uploadedFileCount = 0; - var uploadedTotalBytes = 0; - foreach (var item in items) + uploadedFileCount++; + uploadedTotalBytes += (int)item.fileInfo.Length; + pbar.Tick(uploadedTotalBytes, $"Uploaded {uploadedFileCount} of {items.Count}"); + //if (Interlocked.Read(ref iteration) % 25 == 0) { - var str = $"{item.fileInfo.Name} : 0 of {(int)item.fileInfo.Length.GetSizeInKB()} Kb"; - childPBar = pbar.Spawn((int)item.fileInfo.Length, str, childPbarOptions); + var tsTaken = DateTime.UtcNow.Subtract(dtStart).TotalMilliseconds; + var timePerCombination = tsTaken / uploadedFileCount; + pbar.EstimatedDuration = TimeSpan.FromMilliseconds((items.Count - uploadedFileCount) * timePerCombination); + } + } - //todo: pass Action or Func for a callback instead of raising an event? - var uploadToken = await _googlePhotosSvc.UploadMediaAsync(item.fileInfo.FullName/*, callback: child.Tick()*/); - if (!string.IsNullOrWhiteSpace(uploadToken)) - item.uploadToken = uploadToken; + pbar.Dispose(); + + //album duplicate checking needs to happen first + var requiredAlbumTitles = items.SelectMany(p => p.albums).Distinct(StringComparer.OrdinalIgnoreCase).Where(p => !string.IsNullOrWhiteSpace(p)).ToList(); + //allAlbums = await _diskCacheSvc.GetAsync($"albums.json", () => _googlePhotosSvc.GetAlbumsAsync()); + allAlbums = await _googlePhotosSvc.GetAlbumsAsync(); + if (!requiredAlbumTitles.IsNullOrEmpty()) + DoDuplicateAlbumsExist(); + var dAlbums = await GetOrCreateAlbums(); + if (dAlbums is null) return 1; + + + _console.Write($"Adding {items.Count} media item(s) to your library..."); + var uploadItems = items.Select(p => (p.uploadToken, p.fileInfo.Name)).ToList(); + var res = await _googlePhotosSvc.AddMediaItemsAsync(uploadItems); + if (res is object) + { + _console.WriteLine($" done! :)"); + //iterate over results and assign the MediaItem object to our collection + foreach (var newMediaItem in res.newMediaItemResults) + { + var item = items.FirstOrDefault(p => p.uploadToken == newMediaItem.uploadToken); + if (item is null) + throw new Exception("could this happen?"); + if (newMediaItem.status is object && newMediaItem.status.message == "Success") + item.mediaItem = newMediaItem.mediaItem; else { Debugger.Break(); - //todo: how to handle upload failure here? + //todo: handle error? } + } - childPBar.Dispose(); - - uploadedFileCount++; - uploadedTotalBytes += (int)item.fileInfo.Length; - pbar.Tick(uploadedTotalBytes, $"Uploaded {uploadedFileCount} of {items.Count}"); - //if (Interlocked.Read(ref iteration) % 25 == 0) + //todo: delete local files (if required) + //todo: delete empty folders? + if (deleteLocal) + foreach (var item in items.Where(p => p.mediaItem is object)) { - var tsTaken = DateTime.UtcNow.Subtract(dtStart).TotalMilliseconds; - var timePerCombination = tsTaken / uploadedFileCount; - pbar.EstimatedDuration = TimeSpan.FromMilliseconds((items.Count - uploadedFileCount) * timePerCombination); + _console.Write($"Deleting '{item.fileInfo.FullName}'..."); + File.Delete(item.fileInfo.FullName);//todo: try...catch here? + _console.WriteLine($" deleted!"); } - } - - pbar.Dispose(); - //album duplicate checking needs to happen first - var requiredAlbumTitles = items.SelectMany(p => p.albums).Distinct(StringComparer.OrdinalIgnoreCase).Where(p => !string.IsNullOrWhiteSpace(p)).ToList(); - //allAlbums = await _diskCacheSvc.GetAsync($"albums.json", () => _googlePhotosSvc.GetAlbumsAsync()); - allAlbums = await _googlePhotosSvc.GetAlbumsAsync(); - if (!requiredAlbumTitles.IsNullOrEmpty()) - DoDuplicateAlbumsExist(); - var dAlbums = await GetOrCreateAlbums(); - if (dAlbums is null) return 1; - - - _console.Write($"Adding {items.Count} media item(s) to your library..."); - var uploadItems = items.Select(p => (p.uploadToken, p.fileInfo.Name)).ToList(); - var res = await _googlePhotosSvc.AddMediaItemsAsync(uploadItems); - if (res is object) + if (dAlbums.Count > 0) { - _console.WriteLine($" done! :)"); - //iterate over results and assign the MediaItem object to our collection - foreach (var newMediaItem in res.newMediaItemResults) + _console.WriteLine($"Adding media item(s) to albums..."); + //todo: put progress bar here? + var table = new Table("Album Name", "Status") { Config = TableConfiguration.Markdown() }; + foreach (var kvp in dAlbums) { - var item = items.FirstOrDefault(p => p.uploadToken == newMediaItem.uploadToken); - if (item is null) - throw new Exception("could this happen?"); - if (newMediaItem.status is object && newMediaItem.status.message == "Success") - item.mediaItem = newMediaItem.mediaItem; + var ids = items.Where(p => p.albums.Contains(kvp.Value.title, StringComparer.OrdinalIgnoreCase)).Select(p => p.mediaItem.id).ToList(); + if (await _googlePhotosSvc.AddMediaItemsToAlbumAsync(kvp.Value.id, ids)) + table.AddRow(kvp.Value.title, $"{ids.Count} media item(s) added"); else - { Debugger.Break(); - //todo: handle error? - } } + Console.Write(table.ToString()); + _console.WriteLine(); + } - //todo: delete local files (if required) - //todo: delete empty folders? - if (deleteLocal) - foreach (var item in items.Where(p => p.mediaItem is object)) - { - _console.Write($"Deleting '{item.fileInfo.FullName}'..."); - File.Delete(item.fileInfo.FullName);//todo: try...catch here? - _console.WriteLine($" deleted!"); - } - - if (dAlbums.Count > 0) - { - _console.WriteLine($"Adding media item(s) to albums..."); - //todo: put progress bar here? - var table = new Table("Album Name", "Status") { Config = TableConfiguration.Markdown() }; - foreach (var kvp in dAlbums) - { - var ids = items.Where(p => p.albums.Contains(kvp.Value.title, StringComparer.OrdinalIgnoreCase)).Select(p => p.mediaItem.id).ToList(); - if (await _googlePhotosSvc.AddMediaItemsToAlbumAsync(kvp.Value.id, ids)) - table.AddRow(kvp.Value.title, $"{ids.Count} media item(s) added"); - else - Debugger.Break(); - } - Console.Write(table.ToString()); - _console.WriteLine(); - } + _console.WriteLine($"Upload completed, exiting."); + } + else + _console.WriteLine($" failed :("); + //todo: now handle albums + //todo: do we add media items to local cache here? - _console.WriteLine($"Upload completed, exiting."); - } - else - _console.WriteLine($" failed :("); - //todo: now handle albums - //todo: do we add media items to local cache here? + return 0; - return 0; + bool DoDuplicateAlbumsExist() + { + var duplicateAlbumsByTitle = GetAlbumDuplicates(allAlbums); - bool DoDuplicateAlbumsExist() + //album titles in google photos don't need to be unique, but we can't assign photos to an existing album + //if duplicate titles exist that match one of our required album titles... + if (duplicateAlbumsByTitle.Count > 0 && duplicateAlbumsByTitle.Any(p => requiredAlbumTitles.Contains(p.title, StringComparer.OrdinalIgnoreCase))) { - var duplicateAlbumsByTitle = GetAlbumDuplicates(allAlbums); - - //album titles in google photos don't need to be unique, but we can't assign photos to an existing album - //if duplicate titles exist that match one of our required album titles... - if (duplicateAlbumsByTitle.Count > 0 && duplicateAlbumsByTitle.Any(p => requiredAlbumTitles.Contains(p.title, StringComparer.OrdinalIgnoreCase))) - { - _console.WriteLine($"Duplicate album titles present, unable to assign media item(s) to albums."); - foreach (var album in duplicateAlbumsByTitle) - _console.WriteLine($"{album.title}"); - _console.WriteLine($"Please rename or merge the above albums to continue."); - return false; - } - return true; + _console.WriteLine($"Duplicate album titles present, unable to assign media item(s) to albums."); + foreach (var album in duplicateAlbumsByTitle) + _console.WriteLine($"{album.title}"); + _console.WriteLine($"Please rename or merge the above albums to continue."); + return false; } + return true; + } - async Task> GetOrCreateAlbums() + async Task> GetOrCreateAlbums() + { + //great there are no duplicate titles, lets get/create the missing albums + var d = new Dictionary(); + foreach (var title in requiredAlbumTitles) { - //great there are no duplicate titles, lets get/create the missing albums - var d = new Dictionary(); - foreach (var title in requiredAlbumTitles) - { - var album = allAlbums.FirstOrDefault(p => p.title.Equals(title, StringComparison.OrdinalIgnoreCase)); - if (album is null) - album = await _googlePhotosSvc.CreateAlbumAsync(title); - d.Add(title, album); - } - return d; + var album = allAlbums.FirstOrDefault(p => p.title.Equals(title, StringComparison.OrdinalIgnoreCase)); + if (album is null) + album = await _googlePhotosSvc.CreateAlbumAsync(title); + d.Add(title, album); } + return d; } } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Globals.cs b/src/CasCap.GooglePhotosCli/Globals.cs index e2f0474..6aaec38 100644 --- a/src/CasCap.GooglePhotosCli/Globals.cs +++ b/src/CasCap.GooglePhotosCli/Globals.cs @@ -1,6 +1,4 @@ -using System; - -[Flags] +[Flags] public enum GroupByProperty { filename = 1, diff --git a/src/CasCap.GooglePhotosCli/Models/AppConfig.cs b/src/CasCap.GooglePhotosCli/Models/AppConfig.cs index 0cc2ad9..7f55e39 100644 --- a/src/CasCap.GooglePhotosCli/Models/AppConfig.cs +++ b/src/CasCap.GooglePhotosCli/Models/AppConfig.cs @@ -1,9 +1,7 @@ -using System; -namespace CasCap.Models +namespace CasCap.Models; + +public class AppConfig { - public class AppConfig - { - public DateTime lastCheck { get; set; } = DateTime.MinValue; - public DateTime latestMediaItemCreation { get; set; } = DateTime.MinValue; - } + public DateTime lastCheck { get; set; } = DateTime.MinValue; + public DateTime latestMediaItemCreation { get; set; } = DateTime.MinValue; } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Models/MyMediaFileItem.cs b/src/CasCap.GooglePhotosCli/Models/MyMediaFileItem.cs index b89f322..a24c4f4 100644 --- a/src/CasCap.GooglePhotosCli/Models/MyMediaFileItem.cs +++ b/src/CasCap.GooglePhotosCli/Models/MyMediaFileItem.cs @@ -1,13 +1,12 @@ using System.IO; -namespace CasCap.Models +namespace CasCap.Models; + +public class MyMediaFileItem { - public class MyMediaFileItem - { - public FileInfo fileInfo { get; set; } - public string mimeType { get; set; } - public string relPath { get; set; } - public string[] albums { get; set; } - public string uploadToken { get; set; } - public MediaItem mediaItem { get; set; } - } + public FileInfo fileInfo { get; set; } + public string mimeType { get; set; } + public string relPath { get; set; } + public string[] albums { get; set; } + public string uploadToken { get; set; } + public MediaItem mediaItem { get; set; } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Program.cs b/src/CasCap.GooglePhotosCli/Program.cs index 7b1fe6d..0801a30 100644 --- a/src/CasCap.GooglePhotosCli/Program.cs +++ b/src/CasCap.GooglePhotosCli/Program.cs @@ -3,72 +3,67 @@ using McMaster.Extensions.CommandLineUtils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using System; -using System.Linq; using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -namespace CasCap -{ - [Command(Name = "googlephotos", Description = "*Unofficial* Google Photos CLI", ExtendedHelpText = @" +namespace CasCap; + +[Command(Name = "googlephotos", Description = "*Unofficial* Google Photos CLI", ExtendedHelpText = @" Remarks: See the project site for further information, https://github.com/f2calv/CasCap.GooglePhotosCli ")] - [VersionOptionFromMember("--version", MemberName = nameof(GetVersion))] - [Subcommand(typeof(Logout))] - [Subcommand(typeof(Albums)), Subcommand(typeof(MediaItems))] - [Subcommand(typeof(Sync))] - class Program +[VersionOptionFromMember("--version", MemberName = nameof(GetVersion))] +[Subcommand(typeof(Logout))] +[Subcommand(typeof(Albums)), Subcommand(typeof(MediaItems))] +[Subcommand(typeof(Sync))] +class Program +{ + static async Task Main(string[] args) { - static async Task Main(string[] args) - { - var host = new HostBuilder() - .ConfigureLogging((context, builder) => - { + var host = new HostBuilder() + .ConfigureLogging((context, builder) => + { //builder.AddConsole(); }) - .ConfigureServices((context, services) => - { - services.AddSingleton(); - services.AddSingleton(PhysicalConsole.Singleton); - services.AddGooglePhotos(); - }); - var result = 0; - try + .ConfigureServices((context, services) => { - result = await host.RunCommandLineApplicationAsync(args); - } - catch (CommandParsingException ex) + services.AddSingleton(); + services.AddSingleton(PhysicalConsole.Singleton); + services.AddGooglePhotos(); + }); + var result = 0; + try + { + result = await host.RunCommandLineApplicationAsync(args); + } + catch (CommandParsingException ex) + { + await Console.Error.WriteLineAsync(ex.Message); + if (ex is UnrecognizedCommandParsingException uex && uex.NearestMatches.Any()) { - await Console.Error.WriteLineAsync(ex.Message); - if (ex is UnrecognizedCommandParsingException uex && uex.NearestMatches.Any()) - { - await Console.Error.WriteLineAsync(); - await Console.Error.WriteLineAsync("Did you mean this?"); - await Console.Error.WriteLineAsync(" " + uex.NearestMatches.First()); - } - result = -1; + await Console.Error.WriteLineAsync(); + await Console.Error.WriteLineAsync("Did you mean this?"); + await Console.Error.WriteLineAsync(" " + uex.NearestMatches.First()); } - return result; + result = -1; } + return result; + } - readonly IConsole _console; - readonly GooglePhotosService _googlePhotosSvc; - - public Program(IConsole console, GooglePhotosService googlePhotosSvc) - { - _console = console; - _googlePhotosSvc = googlePhotosSvc; - } + readonly IConsole _console; + readonly GooglePhotosService _googlePhotosSvc; - int OnExecute(CommandLineApplication app, IConsole console, CancellationToken cancellationToken = default) - { - console.WriteLine("You must specify a subcommand."); - app.ShowHelp(); - return 1; - } + public Program(IConsole console, GooglePhotosService googlePhotosSvc) + { + _console = console; + _googlePhotosSvc = googlePhotosSvc; + } - static string GetVersion() - => typeof(Program).Assembly.GetCustomAttribute().InformationalVersion; + int OnExecute(CommandLineApplication app, IConsole console, CancellationToken cancellationToken = default) + { + console.WriteLine("You must specify a subcommand."); + app.ShowHelp(); + return 1; } + + static string GetVersion() + => typeof(Program).Assembly.GetCustomAttribute().InformationalVersion; } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/Services/DiskCacheService.cs b/src/CasCap.GooglePhotosCli/Services/DiskCacheService.cs index d7b2173..0e9137a 100644 --- a/src/CasCap.GooglePhotosCli/Services/DiskCacheService.cs +++ b/src/CasCap.GooglePhotosCli/Services/DiskCacheService.cs @@ -1,108 +1,103 @@ using CasCap.Common.Extensions; using CasCap.Logic; using Microsoft.Extensions.Logging; -using System; using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -namespace CasCap.Services +namespace CasCap.Services; + +public class DiskCacheService { - public class DiskCacheService - { - readonly ILogger _logger; + readonly ILogger _logger; - public DiskCacheService(ILogger logger) - { - _logger = logger; - } + public DiskCacheService(ILogger logger) + { + _logger = logger; + } - readonly AsyncDuplicateLock locker = new(); + readonly AsyncDuplicateLock locker = new(); - public string CacheRoot { get; set; } = string.Empty; + public string CacheRoot { get; set; } = string.Empty; - public bool IsEnabled { get; set; } = true; + public bool IsEnabled { get; set; } = true; - public string CacheSize() + public string CacheSize() + { + var size = Utils.CalculateFolderSize(CacheRoot); + if (size > 1024) { - var size = Utils.CalculateFolderSize(CacheRoot); - if (size > 1024) - { - var s = size / 1024; - return $"{s:###,###,##0}kb"; - } - else - return $"0kb"; + var s = size / 1024; + return $"{s:###,###,##0}kb"; } + else + return $"0kb"; + } - public (int files, int directories) CacheClear() + public (int files, int directories) CacheClear() + { + var di = new DirectoryInfo(CacheRoot); + var files = 0; + foreach (var file in di.GetFiles()) { - var di = new DirectoryInfo(CacheRoot); - var files = 0; - foreach (var file in di.GetFiles()) - { - file.Delete(); - files++; - } - var directories = 0; - foreach (var dir in di.GetDirectories()) - { - dir.Delete(true); - directories++; - } - return (files, directories); + file.Delete(); + files++; } - - public async Task GetAsync(string key, Func> createItem = null, bool skipCache = false, CancellationToken token = default) where T : class + var directories = 0; + foreach (var dir in di.GetDirectories()) { -            //Debug.WriteLine(key); -            var (output, fromCache) = await GetAsyncV2(key, createItem, skipCache, token); - return output; + dir.Delete(true); + directories++; } + return (files, directories); + } -        //V2 returns a boolean indicating the source of the data -        public async Task<(T output, bool fromCache)> GetAsyncV2(string key, Func> createItem = null, bool skipCache = false, CancellationToken token = default) where T : class + public async Task GetAsync(string key, Func> createItem = null, bool skipCache = false, CancellationToken token = default) where T : class + { + //Debug.WriteLine(key); + var (output, fromCache) = await GetAsyncV2(key, createItem, skipCache, token); + return output; + } + + //V2 returns a boolean indicating the source of the data + public async Task<(T output, bool fromCache)> GetAsyncV2(string key, Func> createItem = null, bool skipCache = false, CancellationToken token = default) where T : class + { + var fromCache = false; + key = $"{CacheRoot}/{key}"; + T cacheEntry; + if (IsEnabled && File.Exists(key) && !skipCache) { - var fromCache = false; - key = $"{CacheRoot}/{key}"; - T cacheEntry; - if (IsEnabled && File.Exists(key) && !skipCache) + var json = File.ReadAllText(key); + cacheEntry = null; + try { - var json = File.ReadAllText(key); - cacheEntry = null; - try - { - cacheEntry = json.FromJSON(); - } - catch (Exception ex) - { - Debug.WriteLine(ex); - Debugger.Break(); - } - _logger.LogDebug($"{key}\tretrieved cacheEntry from local cache"); - fromCache = true; + cacheEntry = json.FromJSON(); } - else + catch (Exception ex) { -                //if we use Func and go create the cacheEntry, then we lock here to prevent multiple going at the same time -                //https://www.hanselman.com/blog/EyesWideOpenCorrectCachingIsAlwaysHard.aspx -                using (await locker.LockAsync(key)) - { -                    // Key not in cache, so get data. -                    cacheEntry = await createItem(); - _logger.LogDebug($"{key}\tattempted to populate a new cacheEntry object"); - if (cacheEntry != null && IsEnabled) - File.WriteAllText(key, cacheEntry.ToJSON()); - } + Debug.WriteLine(ex); + Debugger.Break(); } - return (cacheEntry, fromCache); + _logger.LogDebug($"{key}\tretrieved cacheEntry from local cache"); + fromCache = true; } - - public void Delete(string key) + else { - var path = Path.Combine(CacheRoot, key); - if (File.Exists(path)) - File.Delete(path); + //if we use Func and go create the cacheEntry, then we lock here to prevent multiple going at the same time + //https://www.hanselman.com/blog/EyesWideOpenCorrectCachingIsAlwaysHard.aspx + using (await AsyncDuplicateLock.LockAsync(key)) + { + // Key not in cache, so get data. + cacheEntry = await createItem(); + _logger.LogDebug($"{key}\tattempted to populate a new cacheEntry object"); + if (cacheEntry != null && IsEnabled) + File.WriteAllText(key, cacheEntry.ToJSON()); + } } + return (cacheEntry, fromCache); + } + + public void Delete(string key) + { + var path = Path.Combine(CacheRoot, key); + if (File.Exists(path)) + File.Delete(path); } } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/ViewModels/MediaItemScore.cs b/src/CasCap.GooglePhotosCli/ViewModels/MediaItemScore.cs index 19f577f..09c75a2 100644 --- a/src/CasCap.GooglePhotosCli/ViewModels/MediaItemScore.cs +++ b/src/CasCap.GooglePhotosCli/ViewModels/MediaItemScore.cs @@ -1,11 +1,10 @@ -namespace CasCap.ViewModels +namespace CasCap.ViewModels; + +public struct MediaItemScore { - public struct MediaItemScore - { - public int count { get; set; } + public int count { get; set; } - public GroupByProperty propertyMatches { get; set; } + public GroupByProperty propertyMatches { get; set; } - public override string ToString() => $"{count} - {propertyMatches}"; - } + public override string ToString() => $"{count} - {propertyMatches}"; } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/ViewModels/ScoreResponse.cs b/src/CasCap.GooglePhotosCli/ViewModels/ScoreResponse.cs index 6134612..7d92a09 100644 --- a/src/CasCap.GooglePhotosCli/ViewModels/ScoreResponse.cs +++ b/src/CasCap.GooglePhotosCli/ViewModels/ScoreResponse.cs @@ -1,20 +1,19 @@ using System.Collections.Concurrent; -namespace CasCap.ViewModels +namespace CasCap.ViewModels; + +public class ScoreResponse { - public class ScoreResponse - { - /// - /// for each unique image id store the matching property combinations, along with a count. - /// - public ConcurrentDictionary dScore { get; set; } = new ConcurrentDictionary(); + /// + /// for each unique image id store the matching property combinations, along with a count. + /// + public ConcurrentDictionary dScore { get; set; } = new ConcurrentDictionary(); - /// - /// for each unique property combination store the image ids that count as duplicated. - /// - //public ConcurrentDictionary> dScore2 { get; set; } = new ConcurrentDictionary>(); - //public ConcurrentDictionary> dScore2 { get; set; } = new ConcurrentDictionary>(); + /// + /// for each unique property combination store the image ids that count as duplicated. + /// + //public ConcurrentDictionary> dScore2 { get; set; } = new ConcurrentDictionary>(); + //public ConcurrentDictionary> dScore2 { get; set; } = new ConcurrentDictionary>(); - //record a count of matches per enum - public ConcurrentDictionary dStats { get; set; } = new ConcurrentDictionary(); - } + //record a count of matches per enum + public ConcurrentDictionary dStats { get; set; } = new ConcurrentDictionary(); } \ No newline at end of file diff --git a/src/CasCap.GooglePhotosCli/ViewModels/flattened.cs b/src/CasCap.GooglePhotosCli/ViewModels/flattened.cs index a7dd10c..29e1051 100644 --- a/src/CasCap.GooglePhotosCli/ViewModels/flattened.cs +++ b/src/CasCap.GooglePhotosCli/ViewModels/flattened.cs @@ -1,35 +1,33 @@ using CasCap.Models; -using System; -namespace CasCap.ViewModels +namespace CasCap.ViewModels; + +public class flattened { - public class flattened - { - public string id { get; set; } - public string description { get; set; } - //public string productUrl { get; set; }//ignore for the moment? - //public string baseUrl { get; set; }//ignore for the moment? - public string mimeType { get; set; } - //public MediaMetaData mediaMetadata { get; set; }//flattened in properties below - public string filename { get; set; } + public string id { get; set; } + public string description { get; set; } + //public string productUrl { get; set; }//ignore for the moment? + //public string baseUrl { get; set; }//ignore for the moment? + public string mimeType { get; set; } + //public MediaMetaData mediaMetadata { get; set; }//flattened in properties below + public string filename { get; set; } - //public ContributorInfo? contributorInfo { get; set; }//ignore for the moment? + //public ContributorInfo? contributorInfo { get; set; }//ignore for the moment? - public DateTime creationTime { get; set; } - public string width { get; set; } - public string height { get; set; } + public DateTime creationTime { get; set; } + public string width { get; set; } + public string height { get; set; } - public float focalLength { get; set; }//photo - public float apertureFNumber { get; set; }//photo - public int isoEquivalent { get; set; }//photo - public string exposureTime { get; set; }//photo + public float focalLength { get; set; }//photo + public float apertureFNumber { get; set; }//photo + public int isoEquivalent { get; set; }//photo + public string exposureTime { get; set; }//photo - public double fps { get; set; }//video - public string status { get; set; }//video + public double fps { get; set; }//video + public string status { get; set; }//video - public string cameraMake { get; set; }//photo & video - public string cameraModel { get; set; }//photo & video + public string cameraMake { get; set; }//photo & video + public string cameraModel { get; set; }//photo & video - public string[] albumIds { get; set; } - public GooglePhotosContentCategoryType[] contentCategoryTypes { get; set; } - } + public string[] albumIds { get; set; } + public GooglePhotosContentCategoryType[] contentCategoryTypes { get; set; } } \ No newline at end of file