go-s3-uploader/main.go

package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"

	"github.com/joho/godotenv"
)

type UploadPayload struct {
	localPath  string
	remotePath string
	bucketName string
}

func main() {
	endpoint := getEnv("S3_ENDPOINT")
	accessKeyID := getEnv("S3_ACCESS_ID")
	secretAccessKey := getEnv("S3_ACCESS_SECRET")
	useSSL := false

	minioClient, err := minio.New(endpoint, &minio.Options{
		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
		Secure: useSSL,
	})
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("S3 Service online status: %#v\n", minioClient.IsOnline())

	if !minioClient.IsOnline() {
		return
	}

	workingPath, err := os.Getwd()
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Println("Enter the path of the directory you want to upload")
	var providedPath string
	fmt.Scanln(&providedPath)

	isProvidedPathAbsolute := filepath.IsAbs(providedPath)

	var completePath string

	if isProvidedPathAbsolute {
		completePath = providedPath
	} else {
		completePath = filepath.Join(workingPath, providedPath)
	}

	_, fileErr := os.Stat(completePath)
	if os.IsNotExist(fileErr) {
		log.Fatalln("Directory does not exist idiot")
	}

	fmt.Println("Reading directory: " + completePath)
	localFiles, listLocalFilesErr := os.ReadDir(completePath)
	if listLocalFilesErr != nil {
		log.Fatalln(listLocalFilesErr)
	}
	fmt.Printf("Found %d files\n", len(localFiles))

	fmt.Println("Enter Bucket Name: ")
	var bucketName string
	fmt.Scanln(&bucketName)

	bucketExists, bucketExistsErr := minioClient.BucketExists(context.Background(), bucketName)
	if bucketExistsErr != nil {
		log.Fatalln(bucketExistsErr)
	} else if !bucketExists {
		fmt.Printf("Bucket '%s' does not exist\n", bucketName)
	}

	fmt.Println("Enter a prefix to the file names such as a subfolder (enter nothing to leave empty):")
	var prefix string
	fmt.Scanln(&prefix)

	var failedUploadPayloads []UploadPayload

	for _, localFile := range localFiles {
		payload := UploadPayload{
			remotePath: prefix + localFile.Name(),
			localPath:  filepath.Join(completePath, localFile.Name()),
			bucketName: bucketName,
		}

		err := uploadFile(payload, minioClient)
		if err != nil {
			fmt.Println(err)
			failedUploadPayloads = append(failedUploadPayloads, payload)
		}
	}

	if len(failedUploadPayloads) > 0 {
		retryFailedUploads(failedUploadPayloads, minioClient)
	}
}

func getEnv(key string) string {
	err := godotenv.Load()
	if err != nil {
		log.Fatal(fmt.Sprintf("Error loading .env file: %v", err))
	}

	value := os.Getenv(key)
	if value != "" {
		return value
	}
	panic(fmt.Sprintf("Missing environment variable: %s", key))
}

func uploadFile(payload UploadPayload, minioClient *minio.Client) error {
	pauseBetweenUploadsInMilliseconds := 800

	_, readingRemoteFileErr := minioClient.StatObject(context.Background(), payload.bucketName, payload.remotePath, minio.StatObjectOptions{})
	if readingRemoteFileErr == nil {
		fmt.Printf("File already exists: %s\n", payload.remotePath)
		return nil
	}

	localObject, err := os.Open(payload.localPath)
	if err != nil {
		log.Fatalln(err)
	}
	localObjectStat, readingFullLocalFilePathErr := localObject.Stat()
	if readingFullLocalFilePathErr != nil {
		log.Fatalln(err)
	}

	putRemoteObjectInfo, err := minioClient.PutObject(context.Background(), payload.bucketName, payload.remotePath, localObject, localObjectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
	if err != nil {
		localObject.Close()
		return errors.New(fmt.Sprintf("Error uploading file => %s: %s", err.Error(), payload.localPath))
	}

	fmt.Printf("Uploaded file: %s\n", putRemoteObjectInfo.Key)

	localObject.Close()

	time.Sleep(time.Duration(pauseBetweenUploadsInMilliseconds) * time.Millisecond)

	return nil
}

func retryFailedUploads(payloads []UploadPayload, minioClient *minio.Client) {
	fmt.Printf("Retrying %d files", len(payloads))
	for _, p := range payloads {
		err := uploadFile(p, minioClient)
		if err != nil {
			fmt.Println(err.Error())
		}
	}
}