diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fe6a447 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.12 as builder + +WORKDIR /project +COPY main.go go.mod go.sum ./ +COPY downloader ./downloader +ADD vendor ./vendor +ADD version ./version + +# Production-ready build, without debug information specifically for linux +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o=gitmoo-goog -mod=vendor . + + +FROM alpine:3.12 + +# Add CA certificates required for SSL connections +RUN apk add --update --no-cache ca-certificates + +COPY --from=builder /project/gitmoo-goog /usr/local/bin/gitmoo-goog + +RUN mkdir /app +WORKDIR /app +ENTRYPOINT ["/usr/local/bin/gitmoo-goog"] \ No newline at end of file diff --git a/README.md b/README.md index 4e82d4d..8ea895d 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,15 @@ Usage of ./gitmoo-goog: - album download only from this album (use google album id) -folder string - backup folder + backup folder (default current working directory) -force ignore errors, and force working -logfile string log to this file + -credentials-file string + filepath to where the credentials file can be found (default 'credentials.json') + -token-file string + filepath to where the token should be stored (default 'token.json') -loop loops forever (use as daemon) -max int @@ -74,6 +78,8 @@ Usage of ./gitmoo-goog: Time format used for folder paths based on https://golang.org/pkg/time/#Time.Format (default "2016/Janurary") -use-file-name Use file name when uploaded to Google Photos (default off) + -include-exif + Retain EXIF metadata on downloaded images. Location information is not included because Google does not include it. (default off) -download-throttle Rate in KB/sec, to limit downloading of items (default off) -concurrent-downloads @@ -83,10 +89,10 @@ Usage of ./gitmoo-goog: On Linux, running the following is a good practice: ``` -$ ./gitmoo-goog -folder archive -logfile gitmoo.log -loop -throttle 45 & +$ ./gitmoo-goog -folder archive -logfile gitmoo.log -use-file-name -include-exif -loop -throttle 45 & ``` -This will start the process in background, making an API call every 45 seconds, looping forever on all items and saving them to `{pwd}/archive`. +This will start the process in background, making an API call every 45 seconds, looping forever on all items and saving them to `{pwd}/archive`. All images will be downloaded with a filename and metadata as close to original as Google offers through the api. Logfile will be saved as `gitmoo.log`. @@ -104,4 +110,28 @@ To build you may need to specify that module download mode is using a vendor fol ## Testing: -`go test -mod vendor ./...` \ No newline at end of file +`go test -mod vendor ./...` + +## Docker (Linux only) + +You can run gitmoo-goog in Docker. At the moment you have to build the image yourself. After cloning the repo run: + +``` +$ docker build -t dtylman/gitmoo-goog:latest . +``` + +Now run gitmoo-goo in Docker: + +``` +$ docker run -v $(pwd):/app --user=$(id -u):$(id -g) dtylman/gitmoo-goog:latest +``` + +Replace `$(pwd)` with the location of your storage directory on your computer. +Within the storage directory gitmoo-goog expects the `credentials.json` and will place all the downloaded files. + +The part `--user=$(id -u):$(id -g)` ensures that the downloaded files are owned by the user launching the container. + +Configuring additional settings is possible by adding command arguments like so: +``` +$ docker run -v $(pwd):/app --user=$(id -u):$(id -g) dtylman/gitmoo-goog:latest -loop -throttle 45 +``` diff --git a/downloader/downloader.go b/downloader/downloader.go index c415269..4849b85 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -208,7 +209,11 @@ func (d *Downloader) downloadImage(item *LibraryItem, filePath string) error { if strings.HasPrefix(strings.ToLower(item.MediaItem.MimeType), "video") { url = item.MediaItem.BaseUrl + "=dv" } else { - url = fmt.Sprintf("%v=w%v-h%v", item.MediaItem.BaseUrl, item.MediaItem.MediaMetadata.Width, item.MediaItem.MediaMetadata.Height) + if d.Options.IncludeEXIF { + url = fmt.Sprintf("%v=d", item.MediaItem.BaseUrl) + } else { + url = fmt.Sprintf("%v=w%v-h%v", item.MediaItem.BaseUrl, item.MediaItem.MediaMetadata.Width, item.MediaItem.MediaMetadata.Height) + } } output, err := os.Create(filePath) if err != nil { @@ -233,6 +238,18 @@ func (d *Downloader) downloadImage(item *LibraryItem, filePath string) error { return err } + // close file to prevent conflicts with writing new timestamp in next step + output.Close() + + //If timestamp is available, set access time to current timestamp and set modified time to the time the item was first created (not when it was uploaded to Google Photos) + t, err := time.Parse(time.RFC3339, item.MediaMetadata.CreationTime) + if err == nil { + err = os.Chtimes(filePath, time.Now(), t) + if err != nil { + return errors.New("failed writing timestamp to file: " + err.Error()) + } + } + log.Printf("Downloaded '%v' [saved as '%v'] (%v)", item.FileName, item.UsedFileName, humanize.Bytes(uint64(n))) d.stats.UpdateStatsDownloaded(uint64(n), 1) diff --git a/downloader/options.go b/downloader/options.go index 7d25d1b..2322b19 100644 --- a/downloader/options.go +++ b/downloader/options.go @@ -9,6 +9,8 @@ type Options struct { FolderFormat string //UseFileName use file name when uploaded to Google Photos UseFileName bool + //OriginalFiles retain EXIF metadata on downloaded images. Location information is not included. + IncludeEXIF bool //MaxItems how many items to download MaxItems int //number of items to download on per API call @@ -21,4 +23,8 @@ type Options struct { ConcurrentDownloads int //Google photos AlbumID AlbumID string + //CredentialsFile Google API credentials.json file + CredentialsFile string + //TokenFile Google oauth client token.json file + TokenFile string } diff --git a/main.go b/main.go index 5824007..eeb281e 100644 --- a/main.go +++ b/main.go @@ -27,8 +27,7 @@ var options struct { } // Retrieve a token, saves the token, then returns the generated client. -func getClient(config *oauth2.Config) *http.Client { - tokFile := "token.json" +func getClient(config *oauth2.Config, tokFile string) *http.Client { tok, err := tokenFromFile(tokFile) if err != nil { tok = getTokenFromWeb(config) @@ -79,7 +78,7 @@ func saveToken(path string, token *oauth2.Token) { } func process(downloader *downloader.Downloader) error { - b, err := ioutil.ReadFile("credentials.json") + b, err := ioutil.ReadFile(downloader.Options.CredentialsFile) if err != nil { log.Println("Enable photos API here: https://developers.google.com/photos/library/guides/get-started#enable-the-api") return fmt.Errorf("Unable to read client secret file: %v", err) @@ -90,11 +89,11 @@ func process(downloader *downloader.Downloader) error { if err != nil { return fmt.Errorf("Unable to parse client secret file to config: %v", err) } - client := getClient(config) + client := getClient(config, downloader.Options.TokenFile) log.Printf("Connecting ...") srv, err := photoslibrary.New(client) if err != nil { - return fmt.Errorf("Unable to retrieve Sheets client: %v", err) + return fmt.Errorf("Unable to retrieve Google Photos API client: %v", err) } for true { err := downloader.DownloadAll(srv) @@ -126,8 +125,11 @@ func main() { flag.IntVar(&downloader.Options.Throttle, "throttle", 5, "time, in seconds, to wait between API calls") flag.StringVar(&downloader.Options.FolderFormat, "folder-format", filepath.Join("2006", "January"), "time format used for folder paths based on https://golang.org/pkg/time/#Time.Format") flag.BoolVar(&downloader.Options.UseFileName, "use-file-name", false, "use file name when uploaded to Google Photos") + flag.BoolVar(&downloader.Options.IncludeEXIF, "include-exif", false, "retain EXIF metadata on downloaded images. Location information is not included.") flag.Float64Var(&downloader.Options.DownloadThrottle, "download-throttle", 0, "rate in KB/sec, to limit downloading of items") flag.IntVar(&downloader.Options.ConcurrentDownloads, "concurrent-downloads", 5, "number of concurrent item downloads") + flag.StringVar(&downloader.Options.CredentialsFile, "credentials-file", "credentials.json", "filepath to where the credentials file can be found") + flag.StringVar(&downloader.Options.TokenFile, "token-file", "token.json", "filepath to where the token should be stored") flag.Parse() if options.logfile != "" {