package main import ( "context" "errors" "fmt" "log" "os" "path/filepath" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/joho/godotenv" ) type UploadPayload struct { localPath string remotePath string bucketName string } func main() { endpoint := getEnv("S3_ENDPOINT") accessKeyID := getEnv("S3_ACCESS_ID") secretAccessKey := getEnv("S3_ACCESS_SECRET") useSSL := false minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) if err != nil { log.Fatalln(err) } log.Printf("S3 Service online status: %#v\n", minioClient.IsOnline()) if !minioClient.IsOnline() { return } workingPath, err := os.Getwd() if err != nil { log.Fatalln(err) } fmt.Println("Enter the path of the directory you want to upload") var providedPath string fmt.Scanln(&providedPath) isProvidedPathAbsolute := filepath.IsAbs(providedPath) var completePath string if isProvidedPathAbsolute { completePath = providedPath } else { completePath = filepath.Join(workingPath, providedPath) } _, fileErr := os.Stat(completePath) if os.IsNotExist(fileErr) { log.Fatalln("Directory does not exist idiot") } fmt.Println("Reading directory: " + completePath) localFiles, listLocalFilesErr := os.ReadDir(completePath) if listLocalFilesErr != nil { log.Fatalln(listLocalFilesErr) } fmt.Printf("Found %d files\n", len(localFiles)) fmt.Println("Enter Bucket Name: ") var bucketName string fmt.Scanln(&bucketName) bucketExists, bucketExistsErr := minioClient.BucketExists(context.Background(), bucketName) if bucketExistsErr != nil { log.Fatalln(bucketExistsErr) } else if !bucketExists { fmt.Printf("Bucket '%s' does not exist\n", bucketName) } fmt.Println("Enter a prefix to the file names such as a subfolder (enter nothing to leave empty):") var prefix string fmt.Scanln(&prefix) var failedUploadPayloads []UploadPayload for _, localFile := range localFiles { payload := UploadPayload{ remotePath: prefix + localFile.Name(), localPath: filepath.Join(completePath, localFile.Name()), bucketName: bucketName, } err := uploadFile(payload, minioClient) if err != nil { fmt.Println(err) failedUploadPayloads = append(failedUploadPayloads, payload) } } if len(failedUploadPayloads) > 0 { retryFailedUploads(failedUploadPayloads, minioClient) } } func getEnv(key string) string { err := godotenv.Load() if err != nil { log.Fatal(fmt.Sprintf("Error loading .env file: %v", err)) } value := os.Getenv(key) if value != "" { return value } panic(fmt.Sprintf("Missing environment variable: %s", key)) } func uploadFile(payload UploadPayload, minioClient *minio.Client) error { pauseBetweenUploadsInMilliseconds := 800 _, readingRemoteFileErr := minioClient.StatObject(context.Background(), payload.bucketName, payload.remotePath, minio.StatObjectOptions{}) if readingRemoteFileErr == nil { fmt.Printf("File already exists: %s\n", payload.remotePath) return nil } localObject, err := os.Open(payload.localPath) if err != nil { log.Fatalln(err) } localObjectStat, readingFullLocalFilePathErr := localObject.Stat() if readingFullLocalFilePathErr != nil { log.Fatalln(err) } putRemoteObjectInfo, err := minioClient.PutObject(context.Background(), payload.bucketName, payload.remotePath, localObject, localObjectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { localObject.Close() return errors.New(fmt.Sprintf("Error uploading file => %s: %s", err.Error(), payload.localPath)) } fmt.Printf("Uploaded file: %s\n", putRemoteObjectInfo.Key) localObject.Close() time.Sleep(time.Duration(pauseBetweenUploadsInMilliseconds) * time.Millisecond) return nil } func retryFailedUploads(payloads []UploadPayload, minioClient *minio.Client) { fmt.Printf("Retrying %d files", len(payloads)) for _, p := range payloads { err := uploadFile(p, minioClient) if err != nil { fmt.Println(err.Error()) } } }