allow range request for every content type, move lock only when metadata has to be updated

Dockerfile

@@ -18,7 +18,7 @@ ARG PUID=5000 \
     PGID=5000 \
     RUNAS
 
-RUN mkdir -p /tmp/useradd /tmp/empty && \
+RUN mkdir -p /tmp/useradd && \
     if [ ! -z "$RUNAS" ]; then \
     echo "${RUNAS}:x:${PUID}:${PGID}::/nonexistent:/sbin/nologin" >> /tmp/useradd/passwd && \
     echo "${RUNAS}:!:::::::" >> /tmp/useradd/shadow && \
@@ -29,7 +29,6 @@ FROM scratch AS final
 LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>"
 ARG RUNAS
 
-COPY --from=build /tmp/empty /tmp
 COPY --from=build /tmp/useradd/* /etc/
 COPY --from=build --chown=${RUNAS}  /go/bin/transfersh /go/bin/transfersh
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

README.md

@@ -12,7 +12,7 @@ The service at transfersh.com is of unknown origin and reported as cloud malware
 
 ### Upload:
 ```bash
-$ curl -v --upload-file ./hello.txt https://transfer.sh/hello.txt
+$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt
 ```
 
 ### Encrypt & Upload:
@@ -53,9 +53,8 @@ $ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "Max-Days: 1"
 
 The URL used to request the deletion of a file and returned as a response header.
 ```bash
-curl -sD - --upload-file ./hello.txt https://transfer.sh/hello.txt | grep -i -E 'transfer\.sh|x-url-delete'
-x-url-delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
-https://transfer.sh/hello.txt/BAYh0/hello.txt
+curl -sD - --upload-file ./hello https://transfer.sh/hello.txt | grep 'X-Url-Delete'
+X-Url-Delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
 ```
 
 ## Examples
@@ -294,8 +293,9 @@ transfer()
     local upload_files
     local curl_output
     local awk_output
+    local filename
 
-    du -c -k -L "${file_array[@]}" >&2
+    du --total --block-size="K" --dereference "${file_array[@]}" >&2
     # be compatible with "bash"
     if [[ "${ZSH_NAME}" == "zsh" ]]
     then
@@ -311,6 +311,7 @@ transfer()
             # the parameters "--include" and "--form" will suppress the progress bar.
             for file in "${file_array[@]}"
             do
+                filename="${file##*/}"
                 # show delete link and filter out the delete token from the response header after upload.
                 # it is important to save "curl's" "stdout" via a subshell to a variable or redirect it to another command,
                 # which just redirects to "stdout" in order to have a sane output afterwards.
@@ -318,7 +319,7 @@ transfer()
                 # if "stdout" is redirected to something; e.g. ">/dev/null", "tee /dev/null" or "| <some_command>".
                 # the response header is redirected to "stdout", so redirecting "stdout" to "/dev/null" does not make any sense.
                 # redirecting "curl's" "stderr" to "stdout" ("2>&1") will suppress the progress bar.
-                curl_output=$(curl --request PUT --progress-bar --dump-header - --upload-file "${file}" "https://transfer.sh/")
+                curl_output=$(curl --request PUT --progress-bar --dump-header - --upload-file "${file}" "https://transfer.sh/${filename}")
                 awk_output=$(awk \
                     'gsub("\r", "", $0) && tolower($1) ~ /x-url-delete/ \
                     {

cmd/cmd.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/dutchcoders/transfer.sh/server/storage"
 	"log"
 	"os"
 	"strings"
@@ -300,8 +299,8 @@ type Cmd struct {
 	*cli.App
 }
 
-func versionCommand(_ *cli.Context) {
-	fmt.Println(color.YellowString("transfer.sh %s: Easy file sharing from the command line", Version))
+func versionAction(c *cli.Context) {
+	fmt.Println(color.YellowString(fmt.Sprintf("transfer.sh %s: Easy file sharing from the command line", Version)))
 }
 
 // New is the factory for transfer.sh
@@ -319,7 +318,7 @@ func New() *Cmd {
 	app.Commands = []cli.Command{
 		{
 			Name:   "version",
-			Action: versionCommand,
+			Action: versionAction,
 		},
 	}
 
@@ -328,7 +327,7 @@ func New() *Cmd {
 	}
 
 	app.Action = func(c *cli.Context) {
-		var options []server.OptionFn
+		options := []server.OptionFn{}
 		if v := c.String("listener"); v != "" {
 			options = append(options, server.Listener(v))
 		}
@@ -464,42 +463,42 @@ func New() *Cmd {
 				panic("secret-key not set.")
 			} else if bucket := c.String("bucket"); bucket == "" {
 				panic("bucket not set.")
-			} else if store, err := storage.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
+			} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
 				panic(err)
 			} else {
-				options = append(options, server.UseStorage(store))
+				options = append(options, server.UseStorage(storage))
 			}
 		case "gdrive":
-			chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024
+			chunkSize := c.Int("gdrive-chunk-size")
 
 			if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
-				panic("gdrive-client-json-filepath not set.")
+				panic("client-json-filepath not set.")
 			} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
-				panic("gdrive-local-config-path not set.")
+				panic("local-config-path not set.")
 			} else if basedir := c.String("basedir"); basedir == "" {
 				panic("basedir not set.")
-			} else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
+			} else if storage, err := server.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
 				panic(err)
 			} else {
-				options = append(options, server.UseStorage(store))
+				options = append(options, server.UseStorage(storage))
 			}
 		case "storj":
 			if access := c.String("storj-access"); access == "" {
 				panic("storj-access not set.")
 			} else if bucket := c.String("storj-bucket"); bucket == "" {
 				panic("storj-bucket not set.")
-			} else if store, err := storage.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
+			} else if storage, err := server.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
 				panic(err)
 			} else {
-				options = append(options, server.UseStorage(store))
+				options = append(options, server.UseStorage(storage))
 			}
 		case "local":
 			if v := c.String("basedir"); v == "" {
 				panic("basedir not set.")
-			} else if store, err := storage.NewLocalStorage(v, logger); err != nil {
+			} else if storage, err := server.NewLocalStorage(v, logger); err != nil {
 				panic(err)
 			} else {
-				options = append(options, server.UseStorage(store))
+				options = append(options, server.UseStorage(storage))
 			}
 		default:
 			panic("Provider not set or invalid.")

go.mod

@@ -3,21 +3,18 @@ module github.com/dutchcoders/transfer.sh
 go 1.15
 
 require (
-	cloud.google.com/go/compute v1.18.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go v0.77.0 // indirect
 	github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14
 	github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2
 	github.com/aws/aws-sdk-go v1.37.14
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
 	github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329
-	github.com/dutchcoders/transfer.sh-web v0.0.0-20220824020025-7240e75c3bb8
+	github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3
 	github.com/elazarl/go-bindata-assetfs v1.0.1
 	github.com/fatih/color v1.10.0
 	github.com/garyburd/redigo v1.6.2 // indirect
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.2 // indirect
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/securecookie v1.1.1 // indirect
@@ -27,13 +24,12 @@ require (
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
 	github.com/urfave/cli v1.22.5
+	go.opencensus.io v0.22.6 // indirect
 	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
-	golang.org/x/net v0.6.0 // indirect
-	golang.org/x/oauth2 v0.5.0
-	google.golang.org/api v0.109.0
-	google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
+	google.golang.org/api v0.40.0
+	google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
 	storj.io/common v0.0.0-20220405183405-ffdc3ab808c6
 	storj.io/uplink v1.8.2

server/clamav.go

@@ -34,7 +34,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/dutchcoders/go-clamd"
+	clamd "github.com/dutchcoders/go-clamd"
 	"github.com/gorilla/mux"
 )
 

server/handlers.go

@@ -37,7 +37,7 @@ import (
 	"errors"
 	"fmt"
 	"html"
-	htmlTemplate "html/template"
+	html_template "html/template"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -50,15 +50,13 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	textTemplate "text/template"
+	text_template "text/template"
 	"time"
 
-	"github.com/dutchcoders/transfer.sh/server/storage"
-
 	web "github.com/dutchcoders/transfer.sh-web"
 	"github.com/gorilla/mux"
 	"github.com/microcosm-cc/bluemonday"
-	"github.com/russross/blackfriday/v2"
+	blackfriday "github.com/russross/blackfriday/v2"
 	"github.com/skip2/go-qrcode"
 	"golang.org/x/net/idna"
 )
@@ -74,24 +72,24 @@ func stripPrefix(path string) string {
 	return strings.Replace(path, web.Prefix+"/", "", -1)
 }
 
-func initTextTemplates() *textTemplate.Template {
-	templateMap := textTemplate.FuncMap{"format": formatNumber}
+func initTextTemplates() *text_template.Template {
+	templateMap := text_template.FuncMap{"format": formatNumber}
 
 	// Templates with functions available to them
-	var templates = textTemplate.New("").Funcs(templateMap)
+	var templates = text_template.New("").Funcs(templateMap)
 	return templates
 }
 
-func initHTMLTemplates() *htmlTemplate.Template {
-	templateMap := htmlTemplate.FuncMap{"format": formatNumber}
+func initHTMLTemplates() *html_template.Template {
+	templateMap := html_template.FuncMap{"format": formatNumber}
 
 	// Templates with functions available to them
-	var templates = htmlTemplate.New("").Funcs(templateMap)
+	var templates = html_template.New("").Funcs(templateMap)
 
 	return templates
 }
 
-func healthHandler(w http.ResponseWriter, _ *http.Request) {
+func healthHandler(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("Approaching Neutral Zone, all systems normal and functioning."))
 }
 
@@ -139,7 +137,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var templatePath string
-	var content htmlTemplate.HTML
+	var content html_template.HTML
 
 	switch {
 	case strings.HasPrefix(contentType, "image/"):
@@ -152,7 +150,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
 		templatePath = "download.markdown.html"
 
 		var reader io.ReadCloser
-		if reader, _, err = s.storage.Get(r.Context(), token, filename, nil); err != nil {
+		if reader, _, err = s.storage.Get(r.Context(), token, filename); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -167,9 +165,9 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
 		if strings.HasPrefix(contentType, "text/x-markdown") || strings.HasPrefix(contentType, "text/markdown") {
 			unsafe := blackfriday.Run(data)
 			output := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
-			content = htmlTemplate.HTML(output)
+			content = html_template.HTML(output)
 		} else if strings.HasPrefix(contentType, "text/plain") {
-			content = htmlTemplate.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
+			content = html_template.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
 		} else {
 			templatePath = "download.sandbox.html"
 		}
@@ -196,7 +194,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
 
 	data := struct {
 		ContentType    string
-		Content        htmlTemplate.HTML
+		Content        html_template.HTML
 		Filename       string
 		URL            string
 		URLGet         string
@@ -283,7 +281,7 @@ func (s *Server) viewHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (s *Server) notFoundHandler(w http.ResponseWriter, _ *http.Request) {
+func (s *Server) notFoundHandler(w http.ResponseWriter, r *http.Request) {
 	http.Error(w, http.StatusText(404), 404)
 }
 
@@ -304,15 +302,15 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
 
 	responseBody := ""
 
-	for _, fHeaders := range r.MultipartForm.File {
-		for _, fHeader := range fHeaders {
-			filename := sanitize(fHeader.Filename)
-			contentType := mime.TypeByExtension(filepath.Ext(fHeader.Filename))
+	for _, fheaders := range r.MultipartForm.File {
+		for _, fheader := range fheaders {
+			filename := sanitize(fheader.Filename)
+			contentType := mime.TypeByExtension(filepath.Ext(fheader.Filename))
 
 			var f io.Reader
 			var err error
 
-			if f, err = fHeader.Open(); err != nil {
+			if f, err = fheader.Open(); err != nil {
 				s.logger.Printf("%s", err.Error())
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
@@ -461,7 +459,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
 
 	contentLength := r.ContentLength
 
-	defer storage.CloseCheck(r.Body)
+	defer CloseCheck(r.Body.Close)
 
 	file, err := ioutil.TempFile(s.tempPath, "transfer-")
 	defer s.cleanTmpFile(file)
@@ -688,13 +686,10 @@ func (s *Server) unlock(token, filename string) {
 }
 
 func (s *Server) checkMetadata(ctx context.Context, token, filename string, increaseDownload bool) (metadata, error) {
-	s.lock(token, filename)
-	defer s.unlock(token, filename)
-
 	var metadata metadata
 
-	r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
-	defer storage.CloseCheck(r)
+	r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
+	defer CloseCheck(r.Close)
 
 	if err != nil {
 		return metadata, err
@@ -707,7 +702,23 @@ func (s *Server) checkMetadata(ctx context.Context, token, filename string, incr
 	} else if !metadata.MaxDate.IsZero() && time.Now().After(metadata.MaxDate) {
 		return metadata, errors.New("maxDate expired")
 	} else if metadata.MaxDownloads != -1 && increaseDownload {
-		// todo(nl5887): mutex?
+		s.lock(token, filename)
+		defer s.unlock(token, filename)
+
+		r2, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
+		defer CloseCheck(r2.Close)
+
+		if err != nil {
+			return metadata, err
+		}
+
+		if err := json.NewDecoder(r2).Decode(&metadata); err != nil {
+			return metadata, err
+		}
+
+		if metadata.Downloads >= metadata.MaxDownloads {
+			return metadata, errors.New("maxDownloads expired")
+		}
 
 		// update number of downloads
 		metadata.Downloads++
@@ -729,8 +740,8 @@ func (s *Server) checkDeletionToken(ctx context.Context, deletionToken, token, f
 
 	var metadata metadata
 
-	r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
-	defer storage.CloseCheck(r)
+	r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
+	defer CloseCheck(r.Close)
 
 	if s.storage.IsNotExist(err) {
 		return errors.New("metadata doesn't exist")
@@ -807,8 +818,8 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		reader, _, err := s.storage.Get(r.Context(), token, filename, nil)
-		defer storage.CloseCheck(reader)
+		reader, _, err := s.storage.Get(r.Context(), token, filename)
+		defer CloseCheck(reader.Close)
 
 		if err != nil {
 			if s.storage.IsNotExist(err) {
@@ -861,10 +872,10 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
 	commonHeader(w, tarfilename)
 
 	gw := gzip.NewWriter(w)
-	defer storage.CloseCheck(gw)
+	defer CloseCheck(gw.Close)
 
 	zw := tar.NewWriter(gw)
-	defer storage.CloseCheck(zw)
+	defer CloseCheck(zw.Close)
 
 	for _, key := range strings.Split(files, ",") {
 		key = resolveKey(key, s.proxyPath)
@@ -877,8 +888,8 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
-		defer storage.CloseCheck(reader)
+		reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
+		defer CloseCheck(reader.Close)
 
 		if err != nil {
 			if s.storage.IsNotExist(err) {
@@ -922,7 +933,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
 	commonHeader(w, tarfilename)
 
 	zw := tar.NewWriter(w)
-	defer storage.CloseCheck(zw)
+	defer CloseCheck(zw.Close)
 
 	for _, key := range strings.Split(files, ",") {
 		key = resolveKey(key, s.proxyPath)
@@ -935,8 +946,8 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
-		defer storage.CloseCheck(reader)
+		reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
+		defer CloseCheck(reader.Close)
 
 		if err != nil {
 			if s.storage.IsNotExist(err) {
@@ -996,15 +1007,12 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
 
 	remainingDownloads, remainingDays := metadata.remainingLimitHeaderValues()
 
+	w.Header().Set("Accept-Ranges", "bytes")
 	w.Header().Set("Content-Type", contentType)
 	w.Header().Set("Content-Length", strconv.FormatUint(contentLength, 10))
 	w.Header().Set("Connection", "close")
 	w.Header().Set("X-Remaining-Downloads", remainingDownloads)
 	w.Header().Set("X-Remaining-Days", remainingDays)
-
-	if s.storage.IsRangeSupported() {
-		w.Header().Set("Accept-Ranges", "bytes")
-	}
 }
 
 func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
@@ -1022,16 +1030,9 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var rng *storage.Range
-	if r.Header.Get("Range") != "" {
-		rng = storage.ParseRange(r.Header.Get("Range"))
-	}
-
 	contentType := metadata.ContentType
-	reader, contentLength, err := s.storage.Get(r.Context(), token, filename, rng)
-	defer storage.CloseCheck(reader)
-
-	rdr := io.Reader(reader)
+	reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
+	defer CloseCheck(reader.Close)
 
 	if s.storage.IsNotExist(err) {
 		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
@@ -1041,30 +1042,11 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 		return
 	}
-	if rng != nil {
-		cr := rng.ContentRange()
-		if cr != "" {
-			w.Header().Set("Accept-Ranges", "bytes")
-			w.Header().Set("Content-Range", cr)
-			if rng.Limit > 0 {
-				rdr = io.LimitReader(reader, int64(rng.Limit))
-			}
-		}
-	}
 
 	var disposition string
 
 	if action == "inline" {
 		disposition = "inline"
-		/*
-			metadata.ContentType is unable to determine the type of the content,
-			metadata.ContentType is unable to determine the type of the content,
-			metadata.ContentType is unable to determine the type of the content,
-			So add text/plain in this case to fix XSS related issues/
-		*/
-		if strings.TrimSpace(contentType) == "" {
-			contentType = "text/plain"
-		}
 	} else {
 		disposition = "attachment"
 	}
@@ -1079,15 +1061,32 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("X-Remaining-Downloads", remainingDownloads)
 	w.Header().Set("X-Remaining-Days", remainingDays)
 
-	if rng != nil && rng.ContentRange() != "" {
-		w.WriteHeader(http.StatusPartialContent)
-	}
-
 	if disposition == "inline" && canContainsXSS(contentType) {
 		reader = ioutil.NopCloser(bluemonday.UGCPolicy().SanitizeReader(reader))
 	}
 
-	if _, err = io.Copy(w, rdr); err != nil {
+	if r.Header.Get("Range") != "" {
+		file, err := ioutil.TempFile(s.tempPath, "range-")
+		defer s.cleanTmpFile(file)
+
+		if err != nil {
+			s.logger.Printf("%s", err.Error())
+			http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
+			return
+		}
+
+		_, err = io.Copy(file, reader)
+		if err != nil {
+			s.logger.Printf("%s", err.Error())
+			http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
+			return
+		}
+
+		http.ServeContent(w, r, filename, time.Now(), file)
+		return
+	}
+
+	if _, err = io.Copy(w, reader); err != nil {
 		s.logger.Printf("%s", err.Error())
 		http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
 		return

server/ip_filter.go

@@ -88,10 +88,10 @@ func (f *ipFilter) BlockIP(ip string) bool {
 }
 
 func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
-	//check if provided string describes a subnet
-	if ip, network, err := net.ParseCIDR(str); err == nil {
+	//check if has subnet
+	if ip, net, err := net.ParseCIDR(str); err == nil {
 		// containing only one ip?
-		if n, total := network.Mask.Size(); n == total {
+		if n, total := net.Mask.Size(); n == total {
 			f.mut.Lock()
 			f.ips[ip.String()] = allowed
 			f.mut.Unlock()
@@ -110,7 +110,7 @@ func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
 		if !found {
 			f.subnets = append(f.subnets, &subnet{
 				str:     str,
-				ipnet:   network,
+				ipnet:   net,
 				allowed: allowed,
 			})
 		}
@@ -182,7 +182,7 @@ func (f *ipFilter) NetBlocked(ip net.IP) bool {
 	return !f.NetAllowed(ip)
 }
 
-//Wrap the provided handler with simple IP blocking middleware
+//WrapIPFilter the provided handler with simple IP blocking middleware
 //using this IP filter and its configuration
 func (f *ipFilter) Wrap(next http.Handler) http.Handler {
 	return &ipFilterMiddleware{ipFilter: f, next: next}

server/server.go

@@ -26,7 +26,7 @@ package server
 
 import (
 	"context"
-	cryptoRand "crypto/rand"
+	crypto_rand "crypto/rand"
 	"crypto/tls"
 	"encoding/binary"
 	"errors"
@@ -52,7 +52,6 @@ import (
 	"golang.org/x/crypto/acme/autocert"
 
 	web "github.com/dutchcoders/transfer.sh-web"
-	"github.com/dutchcoders/transfer.sh/server/storage"
 	assetfs "github.com/elazarl/go-bindata-assetfs"
 )
 
@@ -244,7 +243,7 @@ func EnableProfiler() OptionFn {
 }
 
 // UseStorage set storage to use
-func UseStorage(s storage.Storage) OptionFn {
+func UseStorage(s Storage) OptionFn {
 	return func(srvr *Server) {
 		srvr.storage = s
 	}
@@ -333,7 +332,7 @@ type Server struct {
 	purgeDays     time.Duration
 	purgeInterval time.Duration
 
-	storage storage.Storage
+	storage Storage
 
 	forceHTTPS bool
 
@@ -381,7 +380,7 @@ func New(options ...OptionFn) (*Server, error) {
 
 func init() {
 	var seedBytes [8]byte
-	if _, err := cryptoRand.Read(seedBytes[:]); err != nil {
+	if _, err := crypto_rand.Read(seedBytes[:]); err != nil {
 		panic("cannot obtain cryptographically secure seed")
 	}
 	rand.Seed(int64(binary.LittleEndian.Uint64(seedBytes[:])))
@@ -475,7 +474,7 @@ func (s *Server) Run() {
 			return false
 		}
 
-		match = r.Referer() == ""
+		match = (r.Referer() == "")
 
 		u, err := url.Parse(r.Referer())
 		if err != nil {
@@ -537,34 +536,32 @@ func (s *Server) Run() {
 	)
 
 	if !s.TLSListenerOnly {
+		srvr := &http.Server{
+			Addr:    s.ListenerString,
+			Handler: h,
+		}
+
 		listening = true
-		s.logger.Printf("starting to listen on: %v\n", s.ListenerString)
+		s.logger.Printf("listening on port: %v\n", s.ListenerString)
 
 		go func() {
-			srvr := &http.Server{
-				Addr:    s.ListenerString,
-				Handler: h,
-			}
-
-			if err := srvr.ListenAndServe(); err != nil {
-				s.logger.Fatal(err)
-			}
+			_ = srvr.ListenAndServe()
 		}()
 	}
 
 	if s.TLSListenerString != "" {
 		listening = true
-		s.logger.Printf("starting to listen for TLS on: %v\n", s.TLSListenerString)
+		s.logger.Printf("listening on port: %v\n", s.TLSListenerString)
 
 		go func() {
-			srvr := &http.Server{
+			s := &http.Server{
 				Addr:      s.TLSListenerString,
 				Handler:   h,
 				TLSConfig: s.tlsConfig,
 			}
 
-			if err := srvr.ListenAndServeTLS("", ""); err != nil {
-				s.logger.Fatal(err)
+			if err := s.ListenAndServeTLS("", ""); err != nil {
+				panic(err)
 			}
 		}()
 	}

server/storage.go

@@ -0,0 +1,812 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	drive "google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
+
+	"storj.io/common/fpath"
+	"storj.io/common/storj"
+	"storj.io/uplink"
+)
+
+// Storage is the interface for storage operation
+type Storage interface {
+	// Get retrieves a file from storage
+	Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error)
+	// Head retrieves content length of a file from storage
+	Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
+	// Put saves a file on storage
+	Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
+	// Delete removes a file from storage
+	Delete(ctx context.Context, token string, filename string) error
+	// IsNotExist indicates if a file doesn't exist on storage
+	IsNotExist(err error) bool
+	// Purge cleans up the storage
+	Purge(ctx context.Context, days time.Duration) error
+
+	// Type returns the storage type
+	Type() string
+}
+
+// LocalStorage is a local storage
+type LocalStorage struct {
+	Storage
+	basedir string
+	logger  *log.Logger
+}
+
+// NewLocalStorage is the factory for LocalStorage
+func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
+	return &LocalStorage{basedir: basedir, logger: logger}, nil
+}
+
+// Type returns the storage type
+func (s *LocalStorage) Type() string {
+	return "local"
+}
+
+// Head retrieves content length of a file from storage
+func (s *LocalStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
+	path := filepath.Join(s.basedir, token, filename)
+
+	var fi os.FileInfo
+	if fi, err = os.Lstat(path); err != nil {
+		return
+	}
+
+	contentLength = uint64(fi.Size())
+
+	return
+}
+
+// Get retrieves a file from storage
+func (s *LocalStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
+	path := filepath.Join(s.basedir, token, filename)
+
+	// content type , content length
+	if reader, err = os.Open(path); err != nil {
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = os.Lstat(path); err != nil {
+		return
+	}
+
+	contentLength = uint64(fi.Size())
+
+	return
+}
+
+// Delete removes a file from storage
+func (s *LocalStorage) Delete(ctx context.Context, token string, filename string) (err error) {
+	metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
+	_ = os.Remove(metadata)
+
+	path := filepath.Join(s.basedir, token, filename)
+	err = os.Remove(path)
+	return
+}
+
+// Purge cleans up the storage
+func (s *LocalStorage) Purge(ctx context.Context, days time.Duration) (err error) {
+	err = filepath.Walk(s.basedir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			if info.ModTime().Before(time.Now().Add(-1 * days)) {
+				err = os.Remove(path)
+				return err
+			}
+
+			return nil
+		})
+
+	return
+}
+
+// IsNotExist indicates if a file doesn't exist on storage
+func (s *LocalStorage) IsNotExist(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	return os.IsNotExist(err)
+}
+
+// Put saves a file on storage
+func (s *LocalStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
+	var f io.WriteCloser
+	var err error
+
+	path := filepath.Join(s.basedir, token)
+
+	if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	defer CloseCheck(f.Close)
+
+	if err != nil {
+		return err
+	}
+
+	if _, err = io.Copy(f, reader); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// S3Storage is a storage backed by AWS S3
+type S3Storage struct {
+	Storage
+	bucket      string
+	session     *session.Session
+	s3          *s3.S3
+	logger      *log.Logger
+	purgeDays   time.Duration
+	noMultipart bool
+}
+
+// NewS3Storage is the factory for S3Storage
+func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
+	sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
+
+	return &S3Storage{
+		bucket:      bucketName,
+		s3:          s3.New(sess),
+		session:     sess,
+		logger:      logger,
+		noMultipart: disableMultipart,
+		purgeDays:   time.Duration(purgeDays*24) * time.Hour,
+	}, nil
+}
+
+// Type returns the storage type
+func (s *S3Storage) Type() string {
+	return "s3"
+}
+
+// Head retrieves content length of a file from storage
+func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
+	key := fmt.Sprintf("%s/%s", token, filename)
+
+	headRequest := &s3.HeadObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(key),
+	}
+
+	// content type , content length
+	response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
+	if err != nil {
+		return
+	}
+
+	if response.ContentLength != nil {
+		contentLength = uint64(*response.ContentLength)
+	}
+
+	return
+}
+
+// Purge cleans up the storage
+func (s *S3Storage) Purge(ctx context.Context, days time.Duration) (err error) {
+	// NOOP expiration is set at upload time
+	return nil
+}
+
+// IsNotExist indicates if a file doesn't exist on storage
+func (s *S3Storage) IsNotExist(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if aerr, ok := err.(awserr.Error); ok {
+		switch aerr.Code() {
+		case s3.ErrCodeNoSuchKey:
+			return true
+		}
+	}
+
+	return false
+}
+
+// Get retrieves a file from storage
+func (s *S3Storage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
+	key := fmt.Sprintf("%s/%s", token, filename)
+
+	getRequest := &s3.GetObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(key),
+	}
+
+	response, err := s.s3.GetObjectWithContext(ctx, getRequest)
+	if err != nil {
+		return
+	}
+
+	if response.ContentLength != nil {
+		contentLength = uint64(*response.ContentLength)
+	}
+
+	reader = response.Body
+	return
+}
+
+// Delete removes a file from storage
+func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
+	metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
+	deleteRequest := &s3.DeleteObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(metadata),
+	}
+
+	_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
+	if err != nil {
+		return
+	}
+
+	key := fmt.Sprintf("%s/%s", token, filename)
+	deleteRequest = &s3.DeleteObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(key),
+	}
+
+	_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
+
+	return
+}
+
+// Put saves a file on storage
+func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
+	key := fmt.Sprintf("%s/%s", token, filename)
+
+	s.logger.Printf("Uploading file %s to S3 Bucket", filename)
+	var concurrency int
+	if !s.noMultipart {
+		concurrency = 20
+	} else {
+		concurrency = 1
+	}
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
+		u.Concurrency = concurrency // default is 5
+		u.LeavePartsOnError = false
+	})
+
+	var expire *time.Time
+	if s.purgeDays.Hours() > 0 {
+		expire = aws.Time(time.Now().Add(s.purgeDays))
+	}
+
+	_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+		Bucket:  aws.String(s.bucket),
+		Key:     aws.String(key),
+		Body:    reader,
+		Expires: expire,
+	})
+
+	return
+}
+
+// GDrive is a storage backed by GDrive
+type GDrive struct {
+	service         *drive.Service
+	rootID          string
+	basedir         string
+	localConfigPath string
+	chunkSize       int
+	logger          *log.Logger
+}
+
+// NewGDriveStorage is the factory for GDrive
+func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
+	b, err := ioutil.ReadFile(clientJSONFilepath)
+	if err != nil {
+		return nil, err
+	}
+
+	// If modifying these scopes, delete your previously saved client_secret.json.
+	config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
+	if err != nil {
+		return nil, err
+	}
+
+	// ToDo: Upgrade deprecated version
+	srv, err := drive.New(getGDriveClient(context.TODO(), config, localConfigPath, logger)) // nolint: staticcheck
+	if err != nil {
+		return nil, err
+	}
+
+	chunkSize = chunkSize * 1024 * 1024
+	storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
+	err = storage.setupRoot()
+	if err != nil {
+		return nil, err
+	}
+
+	return storage, nil
+}
+
+const gdriveRootConfigFile = "root_id.conf"
+const gdriveTokenJSONFile = "token.json"
+const gdriveDirectoryMimeType = "application/vnd.google-apps.folder"
+
+func (s *GDrive) setupRoot() error {
+	rootFileConfig := filepath.Join(s.localConfigPath, gdriveRootConfigFile)
+
+	rootID, err := ioutil.ReadFile(rootFileConfig)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if string(rootID) != "" {
+		s.rootID = string(rootID)
+		return nil
+	}
+
+	dir := &drive.File{
+		Name:     s.basedir,
+		MimeType: gdriveDirectoryMimeType,
+	}
+
+	di, err := s.service.Files.Create(dir).Fields("id").Do()
+	if err != nil {
+		return err
+	}
+
+	s.rootID = di.Id
+	err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *GDrive) hasChecksum(f *drive.File) bool {
+	return f.Md5Checksum != ""
+}
+
+func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
+	return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
+}
+
+func (s *GDrive) findID(filename string, token string) (string, error) {
+	filename = strings.Replace(filename, `'`, `\'`, -1)
+	filename = strings.Replace(filename, `"`, `\"`, -1)
+
+	fileID, tokenID, nextPageToken := "", "", ""
+
+	q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gdriveDirectoryMimeType)
+	l, err := s.list(nextPageToken, q)
+	if err != nil {
+		return "", err
+	}
+
+	for 0 < len(l.Files) {
+		for _, fi := range l.Files {
+			tokenID = fi.Id
+			break
+		}
+
+		if l.NextPageToken == "" {
+			break
+		}
+
+		l, err = s.list(l.NextPageToken, q)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if filename == "" {
+		return tokenID, nil
+	} else if tokenID == "" {
+		return "", fmt.Errorf("cannot find file %s/%s", token, filename)
+	}
+
+	q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gdriveDirectoryMimeType)
+	l, err = s.list(nextPageToken, q)
+	if err != nil {
+		return "", err
+	}
+
+	for 0 < len(l.Files) {
+		for _, fi := range l.Files {
+
+			fileID = fi.Id
+			break
+		}
+
+		if l.NextPageToken == "" {
+			break
+		}
+
+		l, err = s.list(l.NextPageToken, q)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if fileID == "" {
+		return "", fmt.Errorf("cannot find file %s/%s", token, filename)
+	}
+
+	return fileID, nil
+}
+
+// Type returns the storage type
+func (s *GDrive) Type() string {
+	return "gdrive"
+}
+
+// Head retrieves content length of a file from storage
+func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
+	var fileID string
+	fileID, err = s.findID(filename, token)
+	if err != nil {
+		return
+	}
+
+	var fi *drive.File
+	if fi, err = s.service.Files.Get(fileID).Fields("size").Do(); err != nil {
+		return
+	}
+
+	contentLength = uint64(fi.Size)
+
+	return
+}
+
+// Get retrieves a file from storage
+func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
+	var fileID string
+	fileID, err = s.findID(filename, token)
+	if err != nil {
+		return
+	}
+
+	var fi *drive.File
+	fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
+	if err != nil {
+		return
+	}
+	if !s.hasChecksum(fi) {
+		err = fmt.Errorf("cannot find file %s/%s", token, filename)
+		return
+	}
+
+	contentLength = uint64(fi.Size)
+
+	var res *http.Response
+	res, err = s.service.Files.Get(fileID).Context(ctx).Download()
+	if err != nil {
+		return
+	}
+
+	reader = res.Body
+
+	return
+}
+
+// Delete removes a file from storage
+func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
+	metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
+	_ = s.service.Files.Delete(metadata).Do()
+
+	var fileID string
+	fileID, err = s.findID(filename, token)
+	if err != nil {
+		return
+	}
+
+	err = s.service.Files.Delete(fileID).Do()
+	return
+}
+
+// Purge cleans up the storage
+func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
+	nextPageToken := ""
+
+	expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
+	q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gdriveDirectoryMimeType)
+	l, err := s.list(nextPageToken, q)
+	if err != nil {
+		return err
+	}
+
+	for 0 < len(l.Files) {
+		for _, fi := range l.Files {
+			err = s.service.Files.Delete(fi.Id).Do()
+			if err != nil {
+				return
+			}
+		}
+
+		if l.NextPageToken == "" {
+			break
+		}
+
+		l, err = s.list(l.NextPageToken, q)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// IsNotExist indicates if a file doesn't exist on storage
+func (s *GDrive) IsNotExist(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if e, ok := err.(*googleapi.Error); ok {
+		return e.Code == http.StatusNotFound
+	}
+
+	return false
+}
+
+// Put saves a file on storage
+func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
+	dirID, err := s.findID("", token)
+	if err != nil {
+		return err
+	}
+
+	if dirID == "" {
+		dir := &drive.File{
+			Name:     token,
+			Parents:  []string{s.rootID},
+			MimeType: gdriveDirectoryMimeType,
+			Size:     int64(contentLength),
+		}
+
+		di, err := s.service.Files.Create(dir).Fields("id").Do()
+		if err != nil {
+			return err
+		}
+
+		dirID = di.Id
+	}
+
+	// Instantiate empty drive file
+	dst := &drive.File{
+		Name:     filename,
+		Parents:  []string{dirID},
+		MimeType: contentType,
+	}
+
+	_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Retrieve a token, saves the token, then returns the generated client.
+func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
+	tokenFile := filepath.Join(localConfigPath, gdriveTokenJSONFile)
+	tok, err := gDriveTokenFromFile(tokenFile)
+	if err != nil {
+		tok = getGDriveTokenFromWeb(ctx, config, logger)
+		saveGDriveToken(tokenFile, tok, logger)
+	}
+
+	return config.Client(ctx, tok)
+}
+
+// Request a token from the web, then returns the retrieved token.
+func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
+	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
+	fmt.Printf("Go to the following link in your browser then type the "+
+		"authorization code: \n%v\n", authURL)
+
+	var authCode string
+	if _, err := fmt.Scan(&authCode); err != nil {
+		logger.Fatalf("Unable to read authorization code %v", err)
+	}
+
+	tok, err := config.Exchange(ctx, authCode)
+	if err != nil {
+		logger.Fatalf("Unable to retrieve token from web %v", err)
+	}
+	return tok
+}
+
+// Retrieves a token from a local file.
+func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
+	f, err := os.Open(file)
+	defer CloseCheck(f.Close)
+	if err != nil {
+		return nil, err
+	}
+	tok := &oauth2.Token{}
+	err = json.NewDecoder(f).Decode(tok)
+	return tok, err
+}
+
+// Saves a token to a file path.
+func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
+	logger.Printf("Saving credential file to: %s\n", path)
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	defer CloseCheck(f.Close)
+	if err != nil {
+		logger.Fatalf("Unable to cache oauth token: %v", err)
+	}
+
+	err = json.NewEncoder(f).Encode(token)
+	if err != nil {
+		logger.Fatalf("Unable to encode oauth token: %v", err)
+	}
+}
+
+// StorjStorage is a storage backed by Storj
+type StorjStorage struct {
+	Storage
+	project   *uplink.Project
+	bucket    *uplink.Bucket
+	purgeDays time.Duration
+	logger    *log.Logger
+}
+
+// NewStorjStorage is the factory for StorjStorage
+func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
+	var instance StorjStorage
+	var err error
+
+	pCtx := context.TODO()
+
+	ctx := fpath.WithTempData(pCtx, "", true)
+
+	uplConf := &uplink.Config{
+		UserAgent: "transfer-sh",
+	}
+
+	parsedAccess, err := uplink.ParseAccess(access)
+	if err != nil {
+		return nil, err
+	}
+
+	instance.project, err = uplConf.OpenProject(ctx, parsedAccess)
+	if err != nil {
+		return nil, err
+	}
+
+	instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
+	if err != nil {
+		//Ignoring the error to return the one that occurred first, but try to clean up.
+		_ = instance.project.Close()
+		return nil, err
+	}
+
+	instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
+
+	instance.logger = logger
+
+	return &instance, nil
+}
+
+// Type returns the storage type
+func (s *StorjStorage) Type() string {
+	return "storj"
+}
+
+// Head retrieves content length of a file from storage
+func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
+	key := storj.JoinPaths(token, filename)
+
+	obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
+	if err != nil {
+		return 0, err
+	}
+
+	contentLength = uint64(obj.System.ContentLength)
+
+	return
+}
+
+// Get retrieves a file from storage
+func (s *StorjStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
+	key := storj.JoinPaths(token, filename)
+
+	s.logger.Printf("Getting file %s from Storj Bucket", filename)
+
+	download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, nil)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	contentLength = uint64(download.Info().System.ContentLength)
+
+	reader = download
+	return
+}
+
+// Delete removes a file from storage
+func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
+	key := storj.JoinPaths(token, filename)
+
+	s.logger.Printf("Deleting file %s from Storj Bucket", filename)
+
+	_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
+
+	return
+}
+
+// Purge cleans up the storage
+func (s *StorjStorage) Purge(ctx context.Context, days time.Duration) (err error) {
+	// NOOP expiration is set at upload time
+	return nil
+}
+
+// Put saves a file on storage
+func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
+	key := storj.JoinPaths(token, filename)
+
+	s.logger.Printf("Uploading file %s to Storj Bucket", filename)
+
+	var uploadOptions *uplink.UploadOptions
+	if s.purgeDays.Hours() > 0 {
+		uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
+	}
+
+	writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
+	if err != nil {
+		return err
+	}
+
+	n, err := io.Copy(writer, reader)
+	if err != nil || uint64(n) != contentLength {
+		//Ignoring the error to return the one that occurred first, but try to clean up.
+		_ = writer.Abort()
+		return err
+	}
+	err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
+	if err != nil {
+		//Ignoring the error to return the one that occurred first, but try to clean up.
+		_ = writer.Abort()
+		return err
+	}
+
+	err = writer.Commit()
+	return err
+}
+
+// IsNotExist indicates if a file doesn't exist on storage
+func (s *StorjStorage) IsNotExist(err error) bool {
+	return errors.Is(err, uplink.ErrObjectNotFound)
+}

server/storage/common.go

@@ -1,120 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"strconv"
-	"time"
-
-	"regexp"
-)
-
-type Range struct {
-	Start        uint64
-	Limit        uint64
-	contentRange string
-}
-
-// Range Reconstructs Range header and returns it
-func (r *Range) Range() string {
-	if r.Limit > 0 {
-		return fmt.Sprintf("bytes=%d-%d", r.Start, r.Start+r.Limit-1)
-	} else {
-		return fmt.Sprintf("bytes=%d-", r.Start)
-	}
-}
-
-// AcceptLength Tries to accept given range
-// returns newContentLength if range was satisfied, otherwise returns given contentLength
-func (r *Range) AcceptLength(contentLength uint64) (newContentLength uint64) {
-	newContentLength = contentLength
-	if r.Limit == 0 {
-		r.Limit = newContentLength - r.Start
-	}
-	if contentLength < r.Start {
-		return
-	}
-	if r.Limit > contentLength-r.Start {
-		return
-	}
-	r.contentRange = fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Limit-1, contentLength)
-	newContentLength = r.Limit
-	return
-}
-
-func (r *Range) SetContentRange(cr string) {
-	r.contentRange = cr
-}
-
-// Returns accepted Content-Range header. If range wasn't accepted empty string is returned
-func (r *Range) ContentRange() string {
-	return r.contentRange
-}
-
-var rexp *regexp.Regexp = regexp.MustCompile(`^bytes=([0-9]+)-([0-9]*)$`)
-
-// Parses HTTP Range header and returns struct on success
-// only bytes=start-finish supported
-func ParseRange(rng string) *Range {
-	if rng == "" {
-		return nil
-	}
-
-	matches := rexp.FindAllStringSubmatch(rng, -1)
-	if len(matches) != 1 || len(matches[0]) != 3 {
-		return nil
-	}
-	if len(matches[0][0]) != len(rng) || len(matches[0][1]) == 0 {
-		return nil
-	}
-
-	start, err := strconv.ParseUint(matches[0][1], 10, 64)
-	if err != nil {
-		return nil
-	}
-
-	if len(matches[0][2]) == 0 {
-		return &Range{Start: start, Limit: 0}
-	}
-
-	finish, err := strconv.ParseUint(matches[0][2], 10, 64)
-	if err != nil {
-		return nil
-	}
-	if finish < start || finish+1 < finish {
-		return nil
-	}
-
-	return &Range{Start: start, Limit: finish - start + 1}
-}
-
-// Storage is the interface for storage operation
-type Storage interface {
-	// Get retrieves a file from storage
-	Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error)
-	// Head retrieves content length of a file from storage
-	Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
-	// Put saves a file on storage
-	Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
-	// Delete removes a file from storage
-	Delete(ctx context.Context, token string, filename string) error
-	// IsNotExist indicates if a file doesn't exist on storage
-	IsNotExist(err error) bool
-	// Purge cleans up the storage
-	Purge(ctx context.Context, days time.Duration) error
-	// Whether storage supports Get with Range header
-	IsRangeSupported() bool
-	// Type returns the storage type
-	Type() string
-}
-
-func CloseCheck(c io.Closer) {
-	if c == nil {
-		return
-	}
-
-	if err := c.Close(); err != nil {
-		fmt.Println("Received close error:", err)
-	}
-}

server/storage/gdrive.go

@@ -1,394 +0,0 @@
-package storage
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/google"
-	"google.golang.org/api/drive/v3"
-	"google.golang.org/api/googleapi"
-	"google.golang.org/api/option"
-)
-
-// GDrive is a storage backed by GDrive
-type GDrive struct {
-	service         *drive.Service
-	rootID          string
-	basedir         string
-	localConfigPath string
-	chunkSize       int
-	logger          *log.Logger
-}
-
-const gDriveRootConfigFile = "root_id.conf"
-const gDriveTokenJSONFile = "token.json"
-const gDriveDirectoryMimeType = "application/vnd.google-apps.folder"
-
-// NewGDriveStorage is the factory for GDrive
-func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
-
-	ctx := context.TODO()
-
-	b, err := ioutil.ReadFile(clientJSONFilepath)
-	if err != nil {
-		return nil, err
-	}
-
-	// If modifying these scopes, delete your previously saved client_secret.json.
-	config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
-	if err != nil {
-		return nil, err
-	}
-
-	httpClient := getGDriveClient(ctx, config, localConfigPath, logger)
-
-	srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
-	if err != nil {
-		return nil, err
-	}
-
-	storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
-	err = storage.setupRoot()
-	if err != nil {
-		return nil, err
-	}
-
-	return storage, nil
-}
-
-func (s *GDrive) setupRoot() error {
-	rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile)
-
-	rootID, err := ioutil.ReadFile(rootFileConfig)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	if string(rootID) != "" {
-		s.rootID = string(rootID)
-		return nil
-	}
-
-	dir := &drive.File{
-		Name:     s.basedir,
-		MimeType: gDriveDirectoryMimeType,
-	}
-
-	di, err := s.service.Files.Create(dir).Fields("id").Do()
-	if err != nil {
-		return err
-	}
-
-	s.rootID = di.Id
-	err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *GDrive) hasChecksum(f *drive.File) bool {
-	return f.Md5Checksum != ""
-}
-
-func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
-	return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
-}
-
-func (s *GDrive) findID(filename string, token string) (string, error) {
-	filename = strings.Replace(filename, `'`, `\'`, -1)
-	filename = strings.Replace(filename, `"`, `\"`, -1)
-
-	fileID, tokenID, nextPageToken := "", "", ""
-
-	q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gDriveDirectoryMimeType)
-	l, err := s.list(nextPageToken, q)
-	if err != nil {
-		return "", err
-	}
-
-	for 0 < len(l.Files) {
-		for _, fi := range l.Files {
-			tokenID = fi.Id
-			break
-		}
-
-		if l.NextPageToken == "" {
-			break
-		}
-
-		l, err = s.list(l.NextPageToken, q)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	if filename == "" {
-		return tokenID, nil
-	} else if tokenID == "" {
-		return "", fmt.Errorf("cannot find file %s/%s", token, filename)
-	}
-
-	q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gDriveDirectoryMimeType)
-	l, err = s.list(nextPageToken, q)
-	if err != nil {
-		return "", err
-	}
-
-	for 0 < len(l.Files) {
-		for _, fi := range l.Files {
-
-			fileID = fi.Id
-			break
-		}
-
-		if l.NextPageToken == "" {
-			break
-		}
-
-		l, err = s.list(l.NextPageToken, q)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	if fileID == "" {
-		return "", fmt.Errorf("cannot find file %s/%s", token, filename)
-	}
-
-	return fileID, nil
-}
-
-// Type returns the storage type
-func (s *GDrive) Type() string {
-	return "gdrive"
-}
-
-// Head retrieves content length of a file from storage
-func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
-	var fileID string
-	fileID, err = s.findID(filename, token)
-	if err != nil {
-		return
-	}
-
-	var fi *drive.File
-	if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil {
-		return
-	}
-
-	contentLength = uint64(fi.Size)
-
-	return
-}
-
-// Get retrieves a file from storage
-func (s *GDrive) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
-	var fileID string
-	fileID, err = s.findID(filename, token)
-	if err != nil {
-		return
-	}
-
-	var fi *drive.File
-	fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
-	if err != nil {
-		return
-	}
-	if !s.hasChecksum(fi) {
-		err = fmt.Errorf("cannot find file %s/%s", token, filename)
-		return
-	}
-
-	contentLength = uint64(fi.Size)
-
-	fileGetCall := s.service.Files.Get(fileID)
-	if rng != nil {
-		header := fileGetCall.Header()
-		header.Set("Range", rng.Range())
-	}
-
-	var res *http.Response
-	res, err = fileGetCall.Context(ctx).Download()
-	if err != nil {
-		return
-	}
-
-	if rng != nil {
-		reader = res.Body
-		rng.AcceptLength(contentLength)
-		return
-	}
-
-	reader = res.Body
-
-	return
-}
-
-// Delete removes a file from storage
-func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
-	metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
-	_ = s.service.Files.Delete(metadata).Do()
-
-	var fileID string
-	fileID, err = s.findID(filename, token)
-	if err != nil {
-		return
-	}
-
-	err = s.service.Files.Delete(fileID).Context(ctx).Do()
-	return
-}
-
-// Purge cleans up the storage
-func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
-	nextPageToken := ""
-
-	expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
-	q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gDriveDirectoryMimeType)
-	l, err := s.list(nextPageToken, q)
-	if err != nil {
-		return err
-	}
-
-	for 0 < len(l.Files) {
-		for _, fi := range l.Files {
-			err = s.service.Files.Delete(fi.Id).Context(ctx).Do()
-			if err != nil {
-				return
-			}
-		}
-
-		if l.NextPageToken == "" {
-			break
-		}
-
-		l, err = s.list(l.NextPageToken, q)
-		if err != nil {
-			return
-		}
-	}
-
-	return
-}
-
-// IsNotExist indicates if a file doesn't exist on storage
-func (s *GDrive) IsNotExist(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if e, ok := err.(*googleapi.Error); ok {
-		return e.Code == http.StatusNotFound
-	}
-
-	return false
-}
-
-// Put saves a file on storage
-func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
-	dirID, err := s.findID("", token)
-	if err != nil {
-		return err
-	}
-
-	if dirID == "" {
-		dir := &drive.File{
-			Name:     token,
-			Parents:  []string{s.rootID},
-			MimeType: gDriveDirectoryMimeType,
-		}
-
-		di, err := s.service.Files.Create(dir).Fields("id").Do()
-		if err != nil {
-			return err
-		}
-
-		dirID = di.Id
-	}
-
-	// Instantiate empty drive file
-	dst := &drive.File{
-		Name:     filename,
-		Parents:  []string{dirID},
-		MimeType: contentType,
-	}
-
-	_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *GDrive) IsRangeSupported() bool { return true }
-
-// Retrieve a token, saves the token, then returns the generated client.
-func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
-	tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile)
-	tok, err := gDriveTokenFromFile(tokenFile)
-	if err != nil {
-		tok = getGDriveTokenFromWeb(ctx, config, logger)
-		saveGDriveToken(tokenFile, tok, logger)
-	}
-
-	return config.Client(ctx, tok)
-}
-
-// Request a token from the web, then returns the retrieved token.
-func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
-	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
-	fmt.Printf("Go to the following link in your browser then type the "+
-		"authorization code: \n%v\n", authURL)
-
-	var authCode string
-	if _, err := fmt.Scan(&authCode); err != nil {
-		logger.Fatalf("Unable to read authorization code %v", err)
-	}
-
-	tok, err := config.Exchange(ctx, authCode)
-	if err != nil {
-		logger.Fatalf("Unable to retrieve token from web %v", err)
-	}
-	return tok
-}
-
-// Retrieves a token from a local file.
-func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
-	f, err := os.Open(file)
-	defer CloseCheck(f)
-	if err != nil {
-		return nil, err
-	}
-	tok := &oauth2.Token{}
-	err = json.NewDecoder(f).Decode(tok)
-	return tok, err
-}
-
-// Saves a token to a file path.
-func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
-	logger.Printf("Saving credential file to: %s\n", path)
-	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
-	defer CloseCheck(f)
-	if err != nil {
-		logger.Fatalf("Unable to cache oauth token: %v", err)
-	}
-
-	err = json.NewEncoder(f).Encode(token)
-	if err != nil {
-		logger.Fatalf("Unable to encode oauth token: %v", err)
-	}
-}

server/storage/local.go

@@ -1,138 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-// LocalStorage is a local storage
-type LocalStorage struct {
-	Storage
-	basedir string
-	logger  *log.Logger
-}
-
-// NewLocalStorage is the factory for LocalStorage
-func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
-	return &LocalStorage{basedir: basedir, logger: logger}, nil
-}
-
-// Type returns the storage type
-func (s *LocalStorage) Type() string {
-	return "local"
-}
-
-// Head retrieves content length of a file from storage
-func (s *LocalStorage) Head(_ context.Context, token string, filename string) (contentLength uint64, err error) {
-	path := filepath.Join(s.basedir, token, filename)
-
-	var fi os.FileInfo
-	if fi, err = os.Lstat(path); err != nil {
-		return
-	}
-
-	contentLength = uint64(fi.Size())
-
-	return
-}
-
-// Get retrieves a file from storage
-func (s *LocalStorage) Get(_ context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
-	path := filepath.Join(s.basedir, token, filename)
-
-	var file *os.File
-
-	// content type , content length
-	if file, err = os.Open(path); err != nil {
-		return
-	}
-	reader = file
-
-	var fi os.FileInfo
-	if fi, err = os.Lstat(path); err != nil {
-		return
-	}
-
-	contentLength = uint64(fi.Size())
-	if rng != nil {
-		contentLength = rng.AcceptLength(contentLength)
-		if _, err = file.Seek(int64(rng.Start), 0); err != nil {
-			return
-		}
-	}
-
-	return
-}
-
-// Delete removes a file from storage
-func (s *LocalStorage) Delete(_ context.Context, token string, filename string) (err error) {
-	metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
-	_ = os.Remove(metadata)
-
-	path := filepath.Join(s.basedir, token, filename)
-	err = os.Remove(path)
-	return
-}
-
-// Purge cleans up the storage
-func (s *LocalStorage) Purge(_ context.Context, days time.Duration) (err error) {
-	err = filepath.Walk(s.basedir,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-
-			if info.ModTime().Before(time.Now().Add(-1 * days)) {
-				err = os.Remove(path)
-				return err
-			}
-
-			return nil
-		})
-
-	return
-}
-
-// IsNotExist indicates if a file doesn't exist on storage
-func (s *LocalStorage) IsNotExist(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	return os.IsNotExist(err)
-}
-
-// Put saves a file on storage
-func (s *LocalStorage) Put(_ context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
-	var f io.WriteCloser
-	var err error
-
-	path := filepath.Join(s.basedir, token)
-
-	if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
-		return err
-	}
-
-	f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
-	defer CloseCheck(f)
-
-	if err != nil {
-		return err
-	}
-
-	if _, err = io.Copy(f, reader); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *LocalStorage) IsRangeSupported() bool { return true }

server/storage/s3.go

@@ -1,188 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-)
-
-// S3Storage is a storage backed by AWS S3
-type S3Storage struct {
-	Storage
-	bucket      string
-	session     *session.Session
-	s3          *s3.S3
-	logger      *log.Logger
-	purgeDays   time.Duration
-	noMultipart bool
-}
-
-// NewS3Storage is the factory for S3Storage
-func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
-	sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
-
-	return &S3Storage{
-		bucket:      bucketName,
-		s3:          s3.New(sess),
-		session:     sess,
-		logger:      logger,
-		noMultipart: disableMultipart,
-		purgeDays:   time.Duration(purgeDays*24) * time.Hour,
-	}, nil
-}
-
-// Type returns the storage type
-func (s *S3Storage) Type() string {
-	return "s3"
-}
-
-// Head retrieves content length of a file from storage
-func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
-	key := fmt.Sprintf("%s/%s", token, filename)
-
-	headRequest := &s3.HeadObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	}
-
-	// content type , content length
-	response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
-	if err != nil {
-		return
-	}
-
-	if response.ContentLength != nil {
-		contentLength = uint64(*response.ContentLength)
-	}
-
-	return
-}
-
-// Purge cleans up the storage
-func (s *S3Storage) Purge(context.Context, time.Duration) (err error) {
-	// NOOP expiration is set at upload time
-	return nil
-}
-
-// IsNotExist indicates if a file doesn't exist on storage
-func (s *S3Storage) IsNotExist(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if aerr, ok := err.(awserr.Error); ok {
-		switch aerr.Code() {
-		case s3.ErrCodeNoSuchKey:
-			return true
-		}
-	}
-
-	return false
-}
-
-// Get retrieves a file from storage
-func (s *S3Storage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
-	key := fmt.Sprintf("%s/%s", token, filename)
-
-	getRequest := &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	}
-
-	if rng != nil {
-		getRequest.Range = aws.String(rng.Range())
-	}
-
-	response, err := s.s3.GetObjectWithContext(ctx, getRequest)
-	if err != nil {
-		return
-	}
-
-	if response.ContentLength != nil {
-		contentLength = uint64(*response.ContentLength)
-	}
-	if rng != nil && response.ContentRange != nil {
-		rng.SetContentRange(*response.ContentRange)
-	}
-
-	reader = response.Body
-	return
-}
-
-// Delete removes a file from storage
-func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
-	metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
-	deleteRequest := &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(metadata),
-	}
-
-	_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
-	if err != nil {
-		return
-	}
-
-	key := fmt.Sprintf("%s/%s", token, filename)
-	deleteRequest = &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	}
-
-	_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
-
-	return
-}
-
-// Put saves a file on storage
-func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, _ uint64) (err error) {
-	key := fmt.Sprintf("%s/%s", token, filename)
-
-	s.logger.Printf("Uploading file %s to S3 Bucket", filename)
-	var concurrency int
-	if !s.noMultipart {
-		concurrency = 20
-	} else {
-		concurrency = 1
-	}
-
-	// Create an uploader with the session and custom options
-	uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
-		u.Concurrency = concurrency // default is 5
-		u.LeavePartsOnError = false
-	})
-
-	var expire *time.Time
-	if s.purgeDays.Hours() > 0 {
-		expire = aws.Time(time.Now().Add(s.purgeDays))
-	}
-
-	_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
-		Bucket:      aws.String(s.bucket),
-		Key:         aws.String(key),
-		Body:        reader,
-		Expires:     expire,
-		ContentType: aws.String(contentType),
-	})
-
-	return
-}
-
-func (s *S3Storage) IsRangeSupported() bool { return true }
-
-func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
-	return session.Must(session.NewSession(&aws.Config{
-		Region:           aws.String(region),
-		Endpoint:         aws.String(endpoint),
-		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
-		S3ForcePathStyle: aws.Bool(forcePathStyle),
-	}))
-}

server/storage/storj.go

@@ -1,163 +0,0 @@
-package storage
-
-import (
-	"context"
-	"errors"
-	"io"
-	"log"
-	"time"
-
-	"storj.io/common/fpath"
-	"storj.io/common/storj"
-	"storj.io/uplink"
-)
-
-// StorjStorage is a storage backed by Storj
-type StorjStorage struct {
-	Storage
-	project   *uplink.Project
-	bucket    *uplink.Bucket
-	purgeDays time.Duration
-	logger    *log.Logger
-}
-
-// NewStorjStorage is the factory for StorjStorage
-func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
-	var instance StorjStorage
-	var err error
-
-	pCtx := context.TODO()
-
-	ctx := fpath.WithTempData(pCtx, "", true)
-
-	uplConf := &uplink.Config{
-		UserAgent: "transfer-sh",
-	}
-
-	parsedAccess, err := uplink.ParseAccess(access)
-	if err != nil {
-		return nil, err
-	}
-
-	instance.project, err = uplConf.OpenProject(ctx, parsedAccess)
-	if err != nil {
-		return nil, err
-	}
-
-	instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
-	if err != nil {
-		//Ignoring the error to return the one that occurred first, but try to clean up.
-		_ = instance.project.Close()
-		return nil, err
-	}
-
-	instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
-
-	instance.logger = logger
-
-	return &instance, nil
-}
-
-// Type returns the storage type
-func (s *StorjStorage) Type() string {
-	return "storj"
-}
-
-// Head retrieves content length of a file from storage
-func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
-	key := storj.JoinPaths(token, filename)
-
-	obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
-	if err != nil {
-		return 0, err
-	}
-
-	contentLength = uint64(obj.System.ContentLength)
-
-	return
-}
-
-// Get retrieves a file from storage
-func (s *StorjStorage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
-	key := storj.JoinPaths(token, filename)
-
-	s.logger.Printf("Getting file %s from Storj Bucket", filename)
-
-	options := uplink.DownloadOptions{}
-	if rng != nil {
-		options.Offset = int64(rng.Start)
-		if rng.Limit > 0 {
-			options.Length = int64(rng.Limit)
-		}
-	}
-
-	download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, &options)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	contentLength = uint64(download.Info().System.ContentLength)
-	if rng != nil {
-		contentLength = rng.AcceptLength(contentLength)
-	}
-
-	reader = download
-	return
-}
-
-// Delete removes a file from storage
-func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
-	key := storj.JoinPaths(token, filename)
-
-	s.logger.Printf("Deleting file %s from Storj Bucket", filename)
-
-	_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
-
-	return
-}
-
-// Purge cleans up the storage
-func (s *StorjStorage) Purge(context.Context, time.Duration) (err error) {
-	// NOOP expiration is set at upload time
-	return nil
-}
-
-// Put saves a file on storage
-func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
-	key := storj.JoinPaths(token, filename)
-
-	s.logger.Printf("Uploading file %s to Storj Bucket", filename)
-
-	var uploadOptions *uplink.UploadOptions
-	if s.purgeDays.Hours() > 0 {
-		uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
-	}
-
-	writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
-	if err != nil {
-		return err
-	}
-
-	n, err := io.Copy(writer, reader)
-	if err != nil || uint64(n) != contentLength {
-		//Ignoring the error to return the one that occurred first, but try to clean up.
-		_ = writer.Abort()
-		return err
-	}
-	err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
-	if err != nil {
-		//Ignoring the error to return the one that occurred first, but try to clean up.
-		_ = writer.Abort()
-		return err
-	}
-
-	err = writer.Commit()
-	return err
-}
-
-func (s *StorjStorage) IsRangeSupported() bool { return true }
-
-// IsNotExist indicates if a file doesn't exist on storage
-func (s *StorjStorage) IsNotExist(err error) bool {
-	return errors.Is(err, uplink.ErrObjectNotFound)
-}

server/utils.go

@@ -30,12 +30,25 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"net/mail"
 	"strconv"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/golang/gddo/httputil/header"
 )
 
+func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
+	return session.Must(session.NewSession(&aws.Config{
+		Region:           aws.String(region),
+		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		S3ForcePathStyle: aws.Bool(forcePathStyle),
+	}))
+}
+
 func formatNumber(format string, s uint64) string {
 	return renderFloat(format, float64(s))
 }
@@ -188,6 +201,10 @@ func renderFloat(format string, n float64) string {
 	return signStr + intStr + decimalStr + fracStr
 }
 
+func renderInteger(format string, n int) string {
+	return renderFloat(format, float64(n))
+}
+
 // Request.RemoteAddress contains port, which we want to remove i.e.:
 // "[::1]:58292" => "[::1]"
 func ipAddrFromRemoteAddr(s string) string {
@@ -198,16 +215,45 @@ func ipAddrFromRemoteAddr(s string) string {
 	return s[:idx]
 }
 
+func getIPAddress(r *http.Request) string {
+	hdr := r.Header
+	hdrRealIP := hdr.Get("X-Real-Ip")
+	hdrForwardedFor := hdr.Get("X-Forwarded-For")
+	if hdrRealIP == "" && hdrForwardedFor == "" {
+		return ipAddrFromRemoteAddr(r.RemoteAddr)
+	}
+	if hdrForwardedFor != "" {
+		// X-Forwarded-For is potentially a list of addresses separated with ","
+		parts := strings.Split(hdrForwardedFor, ",")
+		for i, p := range parts {
+			parts[i] = strings.TrimSpace(p)
+		}
+
+		// TODO: should return first non-local address
+		return parts[0]
+	}
+	return hdrRealIP
+}
+
+func encodeRFC2047(s string) string {
+	// use mail's rfc2047 to encode any string
+	addr := mail.Address{
+		Name:    s,
+		Address: "",
+	}
+	return strings.Trim(addr.String(), " <>")
+}
+
 func acceptsHTML(hdr http.Header) bool {
 	actual := header.ParseAccept(hdr, "Accept")
 
 	for _, s := range actual {
 		if s.Value == "text/html" {
-			return true
+			return (true)
 		}
 	}
 
-	return false
+	return (false)
 }
 
 func formatSize(size int64) string {
@@ -233,3 +279,9 @@ func formatSize(size int64) string {
 	getSuffix := suffixes[int(math.Floor(base))]
 	return fmt.Sprintf("%s %s", strconv.FormatFloat(newVal, 'f', -1, 64), getSuffix)
 }
+
+func CloseCheck(f func() error) {
+	if err := f(); err != nil {
+		fmt.Println("Received close error:", err)
+	}
+}

server/virustotal.go

@@ -30,7 +30,7 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/dutchcoders/go-virustotal"
+	virustotal "github.com/dutchcoders/go-virustotal"
 )
 
 func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {