dutchcoders/transfer.sh
1
0
Fork 0
mirror of https://github.com/dutchcoders/transfer.sh.git synced 2026-02-03 14:13:26 +00:00
Code Issues 48 Actions Packages Projects Releases Wiki Activity

Merge pull request #1 from dutchcoders/accept-range

Browse Source 
Accept range
This commit is contained in:
Vladislav Grubov
2023-02-12 12:43:30 +03:00
committed by GitHub
parent 2bda0a1e55 2a29083960
commit 1ac8b1ce26
8 changed files with 772 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/cmd.go
View File

@@ -473,9 +473,9 @@ func New() *Cmd {
chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024
if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
panic("client-json-filepath not set.")
panic("gdrive-client-json-filepath not set.")
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
panic("local-config-path not set.")
panic("gdrive-local-config-path not set.")
} else if basedir := c.String("basedir"); basedir == "" {
panic("basedir not set.")
} else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {

16
go.mod
View File

@@ -3,7 +3,8 @@ module github.com/dutchcoders/transfer.sh
go 1.15
require (
cloud.google.com/go v0.77.0 // indirect
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2
github.com/aws/aws-sdk-go v1.37.14
@@ -15,6 +16,8 @@ require (
github.com/fatih/color v1.10.0
github.com/garyburd/redigo v1.6.2 // indirect
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.2 // indirect
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/gorilla/securecookie v1.1.1 // indirect
@@ -24,12 +27,13 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
github.com/urfave/cli v1.22.5
go.opencensus.io v0.22.6 // indirect
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
google.golang.org/api v0.40.0
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 // indirect
golang.org/x/net v0.6.0 // indirect
golang.org/x/oauth2 v0.5.0
google.golang.org/api v0.109.0
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
storj.io/common v0.0.0-20220405183405-ffdc3ab808c6
storj.io/uplink v1.8.2

707
go.sum
View File

File diff suppressed because it is too large Load Diff

45
server/handlers.go
View File

@@ -461,7 +461,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
contentLength := r.ContentLength
defer storage.CloseCheck(r.Body.Close)
defer storage.CloseCheck(r.Body)
file, err := ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
@@ -694,7 +694,7 @@ func (s *Server) checkMetadata(ctx context.Context, token, filename string, incr
var metadata metadata
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
defer storage.CloseCheck(r.Close)
defer storage.CloseCheck(r)
if err != nil {
return metadata, err
@@ -730,7 +730,7 @@ func (s *Server) checkDeletionToken(ctx context.Context, deletionToken, token, f
var metadata metadata
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
defer storage.CloseCheck(r.Close)
defer storage.CloseCheck(r)
if s.storage.IsNotExist(err) {
return errors.New("metadata doesn't exist")
@@ -808,7 +808,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
}
reader, _, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader.Close)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -861,10 +861,10 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
commonHeader(w, tarfilename)
gw := gzip.NewWriter(w)
defer storage.CloseCheck(gw.Close)
defer storage.CloseCheck(gw)
zw := tar.NewWriter(gw)
defer storage.CloseCheck(zw.Close)
defer storage.CloseCheck(zw)
for _, key := range strings.Split(files, ",") {
key = resolveKey(key, s.proxyPath)
@@ -878,7 +878,7 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
}
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader.Close)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -922,7 +922,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
commonHeader(w, tarfilename)
zw := tar.NewWriter(w)
defer storage.CloseCheck(zw.Close)
defer storage.CloseCheck(zw)
for _, key := range strings.Split(files, ",") {
key = resolveKey(key, s.proxyPath)
@@ -936,7 +936,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
}
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader.Close)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -1029,7 +1029,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
contentType := metadata.ContentType
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, rng)
defer storage.CloseCheck(reader.Close)
defer storage.CloseCheck(reader)
rdr := io.Reader(reader)
@@ -1046,7 +1046,9 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
if cr != "" {
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Range", cr)
rdr = io.LimitReader(reader, int64(rng.Limit))
if rng.Limit > 0 {
rdr = io.LimitReader(reader, int64(rng.Limit))
}
}
}
@@ -1085,27 +1087,6 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
reader = ioutil.NopCloser(bluemonday.UGCPolicy().SanitizeReader(reader))
}
if w.Header().Get("Range") != "" || strings.HasPrefix(metadata.ContentType, "video") || strings.HasPrefix(metadata.ContentType, "audio") {
file, err := ioutil.TempFile(s.tempPath, "range-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
_, err = io.Copy(file, rdr)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
http.ServeContent(w, r, filename, time.Now(), file)
return
}
if _, err = io.Copy(w, rdr); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)

32
server/storage/common.go
View File

@@ -16,22 +16,29 @@ type Range struct {
contentRange string
}
// Reconstructs Range header and returns it
// Range Reconstructs Range header and returns it
func (r *Range) Range() string {
return fmt.Sprintf("bytes=%d-%d", r.Start, r.Start+r.Limit-1)
if r.Limit > 0 {
return fmt.Sprintf("bytes=%d-%d", r.Start, r.Start+r.Limit-1)
} else {
return fmt.Sprintf("bytes=%d-", r.Start)
}
}
// Tries to accept given range
// AcceptLength Tries to accept given range
// returns newContentLength if range was satisfied, otherwise returns given contentLength
func (r *Range) AcceptLength(contentLength uint64) (newContentLength uint64) {
newContentLength = contentLength
if r.Limit == 0 {
r.Limit = newContentLength - r.Start
}
if contentLength < r.Start {
return
}
if r.Limit > contentLength-r.Start {
return
}
r.contentRange = fmt.Sprintf("%d-%d/%d", r.Start, r.Start+r.Limit-1, contentLength)
r.contentRange = fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Limit-1, contentLength)
newContentLength = r.Limit
return
}
@@ -45,7 +52,7 @@ func (r *Range) ContentRange() string {
return r.contentRange
}
var rexp *regexp.Regexp = regexp.MustCompile(`^bytes=([0-9]+)-([0-9]+)$`)
var rexp *regexp.Regexp = regexp.MustCompile(`^bytes=([0-9]+)-([0-9]*)$`)
// Parses HTTP Range header and returns struct on success
// only bytes=start-finish supported
@@ -58,7 +65,7 @@ func ParseRange(rng string) *Range {
if len(matches) != 1 || len(matches[0]) != 3 {
return nil
}
if len(matches[0][0]) != len(rng) || len(matches[0][1]) == 0 || len(matches[0][2]) == 0 {
if len(matches[0][0]) != len(rng) || len(matches[0][1]) == 0 {
return nil
}
@@ -66,6 +73,11 @@ func ParseRange(rng string) *Range {
if err != nil {
return nil
}
if len(matches[0][2]) == 0 {
return &Range{Start: start, Limit: 0}
}
finish, err := strconv.ParseUint(matches[0][2], 10, 64)
if err != nil {
return nil
@@ -97,8 +109,12 @@ type Storage interface {
Type() string
}
func CloseCheck(f func() error) {
if err := f(); err != nil {
func CloseCheck(c io.Closer) {
if c == nil {
return
}
if err := c.Close(); err != nil {
fmt.Println("Received close error:", err)
}
}

23
server/storage/gdrive.go
View File

@@ -194,7 +194,7 @@ func (s *GDrive) Head(ctx context.Context, token string, filename string) (conte
}
// Get retrieves a file from storage
func (s *GDrive) Get(ctx context.Context, token string, filename string, _ *Range) (reader io.ReadCloser, contentLength uint64, err error) {
func (s *GDrive) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
@@ -213,12 +213,24 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string, _ *Rang
contentLength = uint64(fi.Size)
fileGetCall := s.service.Files.Get(fileID)
if rng != nil {
header := fileGetCall.Header()
header.Set("Range", rng.Range())
}
var res *http.Response
res, err = s.service.Files.Get(fileID).Context(ctx).Download()
res, err = fileGetCall.Context(ctx).Download()
if err != nil {
return
}
if rng != nil {
reader = res.Body
rng.AcceptLength(contentLength)
return
}
reader = res.Body
return
@@ -296,7 +308,6 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
Name: token,
Parents: []string{s.rootID},
MimeType: gDriveDirectoryMimeType,
Size: int64(contentLength),
}
di, err := s.service.Files.Create(dir).Fields("id").Do()
@@ -323,7 +334,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
return nil
}
func (s *GDrive) IsRangeSupported() bool { return false }
func (s *GDrive) IsRangeSupported() bool { return true }
// Retrieve a token, saves the token, then returns the generated client.
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
@@ -358,7 +369,7 @@ func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *l
// Retrieves a token from a local file.
func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
defer CloseCheck(f.Close)
defer CloseCheck(f)
if err != nil {
return nil, err
}
@@ -371,7 +382,7 @@ func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
logger.Printf("Saving credential file to: %s\n", path)
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f.Close)
defer CloseCheck(f)
if err != nil {
logger.Fatalf("Unable to cache oauth token: %v", err)
}

2
server/storage/local.go
View File

@@ -122,7 +122,7 @@ func (s *LocalStorage) Put(_ context.Context, token string, filename string, rea
}
f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f.Close)
defer CloseCheck(f)
if err != nil {
return err

4
server/storage/storj.go
View File

@@ -86,7 +86,9 @@ func (s *StorjStorage) Get(ctx context.Context, token string, filename string, r
options := uplink.DownloadOptions{}
if rng != nil {
options.Offset = int64(rng.Start)
options.Length = int64(rng.Limit)
if rng.Limit > 0 {
options.Length = int64(rng.Limit)
}
}
download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, &options)