feat: init commit
This commit is contained in:
commit
7da985e3a2
3
.env.example
Normal file
3
.env.example
Normal file
@ -0,0 +1,3 @@
|
||||
S3_ENDPOINT=string
|
||||
S3_ACCESS_ID=string
|
||||
S3_ACCESS_SECRET=string
|
||||
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
.env
|
||||
/build
|
||||
54
README.md
Normal file
54
README.md
Normal file
@ -0,0 +1,54 @@
|
||||
# S3 Directory Uploader
|
||||
|
||||
A simple TUI to upload the files of a directory into an S3 bucket.
|
||||
|
||||
## Setup
|
||||
|
||||
|
||||
```bash
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Set your environment variables to connect to your s3 bucket.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
## Usage
|
||||
Run the program with:
|
||||
```bash
|
||||
go run .
|
||||
```
|
||||
or build and run:
|
||||
```bash
|
||||
$ go build -o s3uploud main.go
|
||||
$ chmod +x ./s3upload
|
||||
$ ./s3upload
|
||||
```
|
||||
|
||||
You will then be prompted for more details.
|
||||
|
||||
##### Bucket
|
||||
Just the name of the bucket man. Just make sure it exists or you will be insulted.
|
||||
|
||||
##### Directory
|
||||
The path to the directory that you want uploaded. You can do relative or absolute.
|
||||
|
||||
##### Prefix
|
||||
If you want to prepend anything to the key of the object in s3. Remember that
|
||||
directories are not real in s3, but prepending the key with a something like
|
||||
`/my/super/important/stuff/` will kind if act like a directory in the s3 UIs.
|
||||
|
||||
If you are using this prefix to act like a directory path, make sure to **not**
|
||||
start it with a `/` and make sure that you **do** end it with a `/`
|
||||
|
||||
**Make sure to read the section that is literally right under this**
|
||||
|
||||
### Design & Limitations
|
||||
- Does not handle nest directories at the moment.
|
||||
- Will retry failed uploads once, retrys happen in bulk at the end of the process
|
||||
- Does not overwrite files with the existing keys
|
||||
- SSL is not enabled, nor is an env var made for it yet, as I built this to use inside
|
||||
my own network. Set `useSSL` to true if using `https` for you endpoint
|
||||
|
||||
27
go.mod
Normal file
27
go.mod
Normal file
@ -0,0 +1,27 @@
|
||||
module s3upload
|
||||
|
||||
go 1.23.7
|
||||
|
||||
require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/minio/minio-go/v7 v7.0.89
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/minio/crc64nvme v1.0.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.36.0 // indirect
|
||||
golang.org/x/net v0.37.0 // indirect
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
golang.org/x/text v0.23.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
40
go.sum
Normal file
40
go.sum
Normal file
@ -0,0 +1,40 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
|
||||
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.89 h1:hx4xV5wwTUfyv8LarhJAwNecnXpoTsj9v3f3q/ZkiJU=
|
||||
github.com/minio/minio-go/v7 v7.0.89/go.mod h1:2rFnGAp02p7Dddo1Fq4S2wYOfpF0MUTSeLTRC90I204=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
|
||||
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
|
||||
golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
|
||||
golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
||||
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
165
main.go
Normal file
165
main.go
Normal file
@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
type UploadPayload struct {
|
||||
localPath string
|
||||
remotePath string
|
||||
bucketName string
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := getEnv("S3_ENDPOINT")
|
||||
accessKeyID := getEnv("S3_ACCESS_ID")
|
||||
secretAccessKey := getEnv("S3_ACCESS_SECRET")
|
||||
useSSL := false
|
||||
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Printf("S3 Service online status: %#v\n", minioClient.IsOnline())
|
||||
|
||||
if !minioClient.IsOnline() {
|
||||
return
|
||||
}
|
||||
|
||||
workingPath, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
fmt.Println("Enter the path of the directory you want to upload")
|
||||
var providedPath string
|
||||
fmt.Scanln(&providedPath)
|
||||
|
||||
isProvidedPathAbsolute := filepath.IsAbs(providedPath)
|
||||
|
||||
var completePath string
|
||||
|
||||
if isProvidedPathAbsolute {
|
||||
completePath = providedPath
|
||||
} else {
|
||||
completePath = filepath.Join(workingPath, providedPath)
|
||||
}
|
||||
|
||||
_, fileErr := os.Stat(completePath)
|
||||
if os.IsNotExist(fileErr) {
|
||||
log.Fatalln("Directory does not exist idiot")
|
||||
}
|
||||
|
||||
fmt.Println("Reading directory: " + completePath)
|
||||
localFiles, listLocalFilesErr := os.ReadDir(completePath)
|
||||
if listLocalFilesErr != nil {
|
||||
log.Fatalln(listLocalFilesErr)
|
||||
}
|
||||
fmt.Printf("Found %d files\n", len(localFiles))
|
||||
|
||||
fmt.Println("Enter Bucket Name: ")
|
||||
var bucketName string
|
||||
fmt.Scanln(&bucketName)
|
||||
|
||||
bucketExists, bucketExistsErr := minioClient.BucketExists(context.Background(), bucketName)
|
||||
if bucketExistsErr != nil {
|
||||
log.Fatalln(bucketExistsErr)
|
||||
} else if !bucketExists {
|
||||
fmt.Printf("Bucket '%s' does not exist\n", bucketName)
|
||||
}
|
||||
|
||||
fmt.Println("Enter a prefix to the file names such as a subfolder (enter nothing to leave empty):")
|
||||
var prefix string
|
||||
fmt.Scanln(&prefix)
|
||||
|
||||
var failedUploadPayloads []UploadPayload
|
||||
|
||||
for _, localFile := range localFiles {
|
||||
payload := UploadPayload{
|
||||
remotePath: prefix + localFile.Name(),
|
||||
localPath: filepath.Join(completePath, localFile.Name()),
|
||||
bucketName: bucketName,
|
||||
}
|
||||
|
||||
err := uploadFile(payload, minioClient)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
failedUploadPayloads = append(failedUploadPayloads, payload)
|
||||
}
|
||||
}
|
||||
|
||||
if len(failedUploadPayloads) > 0 {
|
||||
retryFailedUploads(failedUploadPayloads, minioClient)
|
||||
}
|
||||
}
|
||||
|
||||
func getEnv(key string) string {
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
log.Fatal(fmt.Sprintf("Error loading .env file: %v", err))
|
||||
}
|
||||
|
||||
value := os.Getenv(key)
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
panic(fmt.Sprintf("Missing environment variable: %s", key))
|
||||
}
|
||||
|
||||
func uploadFile(payload UploadPayload, minioClient *minio.Client) error {
|
||||
pauseBetweenUploadsInMilliseconds := 800
|
||||
|
||||
_, readingRemoteFileErr := minioClient.StatObject(context.Background(), payload.bucketName, payload.remotePath, minio.StatObjectOptions{})
|
||||
if readingRemoteFileErr == nil {
|
||||
fmt.Printf("File already exists: %s\n", payload.remotePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
localObject, err := os.Open(payload.localPath)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
localObjectStat, readingFullLocalFilePathErr := localObject.Stat()
|
||||
if readingFullLocalFilePathErr != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
putRemoteObjectInfo, err := minioClient.PutObject(context.Background(), payload.bucketName, payload.remotePath, localObject, localObjectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
|
||||
if err != nil {
|
||||
localObject.Close()
|
||||
return errors.New(fmt.Sprintf("Error uploading file => %s: %s", err.Error(), payload.localPath))
|
||||
}
|
||||
|
||||
fmt.Printf("Uploaded file: %s\n", putRemoteObjectInfo.Key)
|
||||
|
||||
localObject.Close()
|
||||
|
||||
time.Sleep(time.Duration(pauseBetweenUploadsInMilliseconds) * time.Millisecond)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryFailedUploads(payloads []UploadPayload, minioClient *minio.Client) {
|
||||
fmt.Printf("Retrying %d files", len(payloads))
|
||||
for _, p := range payloads {
|
||||
err := uploadFile(p, minioClient)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user