commit 7da985e3a248d4322b4172caf67d20fbe267a17f Author: Yehoshua Sandler Date: Mon Mar 31 17:47:48 2025 -0500 feat: init commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5e0744d --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +S3_ENDPOINT=string +S3_ACCESS_ID=string +S3_ACCESS_SECRET=string diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cd916e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +/build diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc41dde --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# S3 Directory Uploader + +A simple TUI to upload the files of a directory into an S3 bucket. + +## Setup + + +```bash +go mod tidy +``` + +Set your environment variables to connect to your s3 bucket. + +```bash +cp .env.example .env +``` + +## Usage +Run the program with: +```bash +go run . +``` +or build and run: +```bash +$ go build -o s3uploud main.go +$ chmod +x ./s3upload +$ ./s3upload +``` + +You will then be prompted for more details. + +##### Bucket +Just the name of the bucket man. Just make sure it exists or you will be insulted. + +##### Directory +The path to the directory that you want uploaded. You can do relative or absolute. + +##### Prefix +If you want to prepend anything to the key of the object in s3. Remember that +directories are not real in s3, but prepending the key with a something like +`/my/super/important/stuff/` will kind if act like a directory in the s3 UIs. + +If you are using this prefix to act like a directory path, make sure to **not** +start it with a `/` and make sure that you **do** end it with a `/` + +**Make sure to read the section that is literally right under this** + +### Design & Limitations +- Does not handle nest directories at the moment. +- Will retry failed uploads once, retrys happen in bulk at the end of the process +- Does not overwrite files with the existing keys +- SSL is not enabled, nor is an env var made for it yet, as I built this to use inside +my own network. Set `useSSL` to true if using `https` for you endpoint + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d203f33 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module s3upload + +go 1.23.7 + +require ( + github.com/joho/godotenv v1.5.1 + github.com/minio/minio-go/v7 v7.0.89 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.37.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..889be67 --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.89 h1:hx4xV5wwTUfyv8LarhJAwNecnXpoTsj9v3f3q/ZkiJU= +github.com/minio/minio-go/v7 v7.0.89/go.mod h1:2rFnGAp02p7Dddo1Fq4S2wYOfpF0MUTSeLTRC90I204= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..a8ed457 --- /dev/null +++ b/main.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "github.com/joho/godotenv" +) + +type UploadPayload struct { + localPath string + remotePath string + bucketName string +} + +func main() { + endpoint := getEnv("S3_ENDPOINT") + accessKeyID := getEnv("S3_ACCESS_ID") + secretAccessKey := getEnv("S3_ACCESS_SECRET") + useSSL := false + + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + log.Fatalln(err) + } + + log.Printf("S3 Service online status: %#v\n", minioClient.IsOnline()) + + if !minioClient.IsOnline() { + return + } + + workingPath, err := os.Getwd() + if err != nil { + log.Fatalln(err) + } + + fmt.Println("Enter the path of the directory you want to upload") + var providedPath string + fmt.Scanln(&providedPath) + + isProvidedPathAbsolute := filepath.IsAbs(providedPath) + + var completePath string + + if isProvidedPathAbsolute { + completePath = providedPath + } else { + completePath = filepath.Join(workingPath, providedPath) + } + + _, fileErr := os.Stat(completePath) + if os.IsNotExist(fileErr) { + log.Fatalln("Directory does not exist idiot") + } + + fmt.Println("Reading directory: " + completePath) + localFiles, listLocalFilesErr := os.ReadDir(completePath) + if listLocalFilesErr != nil { + log.Fatalln(listLocalFilesErr) + } + fmt.Printf("Found %d files\n", len(localFiles)) + + fmt.Println("Enter Bucket Name: ") + var bucketName string + fmt.Scanln(&bucketName) + + bucketExists, bucketExistsErr := minioClient.BucketExists(context.Background(), bucketName) + if bucketExistsErr != nil { + log.Fatalln(bucketExistsErr) + } else if !bucketExists { + fmt.Printf("Bucket '%s' does not exist\n", bucketName) + } + + fmt.Println("Enter a prefix to the file names such as a subfolder (enter nothing to leave empty):") + var prefix string + fmt.Scanln(&prefix) + + var failedUploadPayloads []UploadPayload + + for _, localFile := range localFiles { + payload := UploadPayload{ + remotePath: prefix + localFile.Name(), + localPath: filepath.Join(completePath, localFile.Name()), + bucketName: bucketName, + } + + err := uploadFile(payload, minioClient) + if err != nil { + fmt.Println(err) + failedUploadPayloads = append(failedUploadPayloads, payload) + } + } + + if len(failedUploadPayloads) > 0 { + retryFailedUploads(failedUploadPayloads, minioClient) + } +} + +func getEnv(key string) string { + err := godotenv.Load() + if err != nil { + log.Fatal(fmt.Sprintf("Error loading .env file: %v", err)) + } + + value := os.Getenv(key) + if value != "" { + return value + } + panic(fmt.Sprintf("Missing environment variable: %s", key)) +} + +func uploadFile(payload UploadPayload, minioClient *minio.Client) error { + pauseBetweenUploadsInMilliseconds := 800 + + _, readingRemoteFileErr := minioClient.StatObject(context.Background(), payload.bucketName, payload.remotePath, minio.StatObjectOptions{}) + if readingRemoteFileErr == nil { + fmt.Printf("File already exists: %s\n", payload.remotePath) + return nil + } + + localObject, err := os.Open(payload.localPath) + if err != nil { + log.Fatalln(err) + } + localObjectStat, readingFullLocalFilePathErr := localObject.Stat() + if readingFullLocalFilePathErr != nil { + log.Fatalln(err) + } + + putRemoteObjectInfo, err := minioClient.PutObject(context.Background(), payload.bucketName, payload.remotePath, localObject, localObjectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + localObject.Close() + return errors.New(fmt.Sprintf("Error uploading file => %s: %s", err.Error(), payload.localPath)) + } + + fmt.Printf("Uploaded file: %s\n", putRemoteObjectInfo.Key) + + localObject.Close() + + time.Sleep(time.Duration(pauseBetweenUploadsInMilliseconds) * time.Millisecond) + + return nil +} + +func retryFailedUploads(payloads []UploadPayload, minioClient *minio.Client) { + fmt.Printf("Retrying %d files", len(payloads)) + for _, p := range payloads { + err := uploadFile(p, minioClient) + if err != nil { + fmt.Println(err.Error()) + } + } +}