Merge branch 'master' into feature-exit-code

Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.12 as builder
+
+WORKDIR /project
+COPY main.go go.mod go.sum ./
+COPY downloader ./downloader
+ADD vendor ./vendor
+ADD version ./version
+
+# Production-ready build, without debug information specifically for linux
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o=gitmoo-goog -mod=vendor .
+
+
+FROM alpine:3.12
+
+# Add CA certificates required for SSL connections
+RUN apk add --update --no-cache ca-certificates
+
+COPY --from=builder /project/gitmoo-goog /usr/local/bin/gitmoo-goog
+
+RUN mkdir /app
+WORKDIR /app
+ENTRYPOINT ["/usr/local/bin/gitmoo-goog"]

README.md

@@ -57,11 +57,15 @@ Usage of ./gitmoo-goog:
   - album
         download only from this album (use google album id)
   -folder string
-        backup folder
+        backup folder (default current working directory)
   -force
         ignore errors, and force working
   -logfile string
         log to this file
+  -credentials-file string
+        filepath to where the credentials file can be found (default 'credentials.json')
+  -token-file string
+        filepath to where the token should be stored (default 'token.json')
   -loop
         loops forever (use as daemon)
   -max int
@@ -74,6 +78,8 @@ Usage of ./gitmoo-goog:
         Time format used for folder paths based on https://golang.org/pkg/time/#Time.Format (default "2016/Janurary")
   -use-file-name
         Use file name when uploaded to Google Photos (default off)
+  -include-exif
+        Retain EXIF metadata on downloaded images. Location information is not included because Google does not include it. (default off)
   -download-throttle
         Rate in KB/sec, to limit downloading of items (default off)
   -concurrent-downloads
@@ -83,10 +89,10 @@ Usage of ./gitmoo-goog:
 On Linux, running the following is a good practice:
 
 ```
-$ ./gitmoo-goog -folder archive -logfile gitmoo.log -loop -throttle 45 &
+$ ./gitmoo-goog -folder archive -logfile gitmoo.log -use-file-name -include-exif -loop -throttle 45 &
 ```
 
-This will start the process in background, making an API call every 45 seconds, looping forever on all items and saving them to `{pwd}/archive`. 
+This will start the process in background, making an API call every 45 seconds, looping forever on all items and saving them to `{pwd}/archive`. All images will be downloaded with a filename and metadata as close to original as Google offers through the api.
 
 Logfile will be saved as `gitmoo.log`.
 
@@ -104,4 +110,28 @@ To build you may need to specify that module download mode is using a vendor fol
 
 ## Testing:
 
-`go test -mod vendor ./...`
+`go test -mod vendor ./...`
+
+## Docker (Linux only)
+
+You can run gitmoo-goog in Docker. At the moment you have to build the image yourself. After cloning the repo run:
+
+```
+$ docker build -t dtylman/gitmoo-goog:latest .
+```
+
+Now run gitmoo-goo in Docker:
+
+```
+$ docker run -v $(pwd):/app --user=$(id -u):$(id -g) dtylman/gitmoo-goog:latest
+```
+
+Replace `$(pwd)` with the location of your storage directory on your computer.
+Within the storage directory gitmoo-goog expects the `credentials.json` and will place all the downloaded files.
+
+The part `--user=$(id -u):$(id -g)` ensures that the downloaded files are owned by the user launching the container.
+
+Configuring additional settings is possible by adding command arguments like so:
+```
+$ docker run -v $(pwd):/app --user=$(id -u):$(id -g) dtylman/gitmoo-goog:latest -loop -throttle 45
+```

downloader/downloader.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -208,7 +209,11 @@ func (d *Downloader) downloadImage(item *LibraryItem, filePath string) error {
 	if strings.HasPrefix(strings.ToLower(item.MediaItem.MimeType), "video") {
 		url = item.MediaItem.BaseUrl + "=dv"
 	} else {
-		url = fmt.Sprintf("%v=w%v-h%v", item.MediaItem.BaseUrl, item.MediaItem.MediaMetadata.Width, item.MediaItem.MediaMetadata.Height)
+		if d.Options.IncludeEXIF {
+			url = fmt.Sprintf("%v=d", item.MediaItem.BaseUrl)
+		} else {
+			url = fmt.Sprintf("%v=w%v-h%v", item.MediaItem.BaseUrl, item.MediaItem.MediaMetadata.Width, item.MediaItem.MediaMetadata.Height)
+		}
 	}
 	output, err := os.Create(filePath)
 	if err != nil {
@@ -233,6 +238,18 @@ func (d *Downloader) downloadImage(item *LibraryItem, filePath string) error {
 		return err
 	}
 
+	// close file to prevent conflicts with writing new timestamp in next step
+	output.Close()
+
+	//If timestamp is available, set access time to current timestamp and set modified time to the time the item was first created (not when it was uploaded to Google Photos)
+	t, err := time.Parse(time.RFC3339, item.MediaMetadata.CreationTime)
+	if err == nil {
+		err = os.Chtimes(filePath, time.Now(), t)
+		if err != nil {
+			return errors.New("failed writing timestamp to file: " + err.Error())
+		}
+	}
+
 	log.Printf("Downloaded '%v' [saved as '%v'] (%v)", item.FileName, item.UsedFileName, humanize.Bytes(uint64(n)))
 
 	d.stats.UpdateStatsDownloaded(uint64(n), 1)

downloader/options.go

@@ -9,6 +9,8 @@ type Options struct {
 	FolderFormat string
 	//UseFileName use file name when uploaded to Google Photos
 	UseFileName bool
+	//OriginalFiles retain EXIF metadata on downloaded images. Location information is not included.
+	IncludeEXIF bool
 	//MaxItems how many items to download
 	MaxItems int
 	//number of items to download on per API call
@@ -21,4 +23,8 @@ type Options struct {
 	ConcurrentDownloads int
 	//Google photos AlbumID
 	AlbumID string
+	//CredentialsFile Google API credentials.json file
+	CredentialsFile string
+	//TokenFile Google oauth client token.json file
+	TokenFile string
 }

main.go

@@ -27,8 +27,7 @@ var options struct {
 }
 
 // Retrieve a token, saves the token, then returns the generated client.
-func getClient(config *oauth2.Config) *http.Client {
-	tokFile := "token.json"
+func getClient(config *oauth2.Config, tokFile string) *http.Client {
 	tok, err := tokenFromFile(tokFile)
 	if err != nil {
 		tok = getTokenFromWeb(config)
@@ -79,7 +78,7 @@ func saveToken(path string, token *oauth2.Token) {
 }
 
 func process(downloader *downloader.Downloader) error {
-	b, err := ioutil.ReadFile("credentials.json")
+	b, err := ioutil.ReadFile(downloader.Options.CredentialsFile)
 	if err != nil {
 		log.Println("Enable photos API here: https://developers.google.com/photos/library/guides/get-started#enable-the-api")
 		return fmt.Errorf("Unable to read client secret file: %v", err)
@@ -90,11 +89,11 @@ func process(downloader *downloader.Downloader) error {
 	if err != nil {
 		return fmt.Errorf("Unable to parse client secret file to config: %v", err)
 	}
-	client := getClient(config)
+	client := getClient(config, downloader.Options.TokenFile)
 	log.Printf("Connecting ...")
 	srv, err := photoslibrary.New(client)
 	if err != nil {
-		return fmt.Errorf("Unable to retrieve Sheets client: %v", err)
+		return fmt.Errorf("Unable to retrieve Google Photos API client: %v", err)
 	}
 	for true {
 		err := downloader.DownloadAll(srv)
@@ -126,8 +125,11 @@ func main() {
 	flag.IntVar(&downloader.Options.Throttle, "throttle", 5, "time, in seconds, to wait between API calls")
 	flag.StringVar(&downloader.Options.FolderFormat, "folder-format", filepath.Join("2006", "January"), "time format used for folder paths based on https://golang.org/pkg/time/#Time.Format")
 	flag.BoolVar(&downloader.Options.UseFileName, "use-file-name", false, "use file name when uploaded to Google Photos")
+	flag.BoolVar(&downloader.Options.IncludeEXIF, "include-exif", false, "retain EXIF metadata on downloaded images. Location information is not included.")
 	flag.Float64Var(&downloader.Options.DownloadThrottle, "download-throttle", 0, "rate in KB/sec, to limit downloading of items")
 	flag.IntVar(&downloader.Options.ConcurrentDownloads, "concurrent-downloads", 5, "number of concurrent item downloads")
+	flag.StringVar(&downloader.Options.CredentialsFile, "credentials-file", "credentials.json", "filepath to where the credentials file can be found")
+	flag.StringVar(&downloader.Options.TokenFile, "token-file", "token.json", "filepath to where the token should be stored")
 
 	flag.Parse()
 	if options.logfile != "" {