Compare commits

...

29 Commits

Author SHA1 Message Date
Andrea Spacca
2a29083960 handle range with no limit 2023-02-11 11:47:51 +09:00
Andrea Spacca
158e5487ee refactor CloseCheck to avoid panic on nil 2023-02-11 11:47:01 +09:00
Andrea Spacca
806286ab35 refactor CloseCheck to avoid panic on nil, handle range in gdrive storage 2023-02-11 11:46:27 +09:00
Andrea Spacca
d49aee59ba refactor CloseCheck to avoid panic on nil, handle range with no limit 2023-02-11 11:45:57 +09:00
Andrea Spacca
e08225e5f8 refactor CloseCheck to avoid panic on nil, remove range/audio/video special handling on get handler 2023-02-11 11:45:17 +09:00
Andrea Spacca
8597f1d9eb bump gdrive dependecies 2023-02-11 11:44:12 +09:00
Andrea Spacca
9e8ce19cd1 proper param name in error 2023-02-11 11:43:41 +09:00
Vladislav Grubov
2bda0a1e55 Adds 'Accept-Ranges: bytes' header to handlers 2023-02-03 22:45:55 +03:00
Vladislav Grubov
d9369e8b39 Support Range header for GET 2023-01-28 22:23:57 +03:00
mpl
193f944829 server: do not ignore listening errors (#523) 2023-01-06 01:43:57 +01:00
Frankie
ebc4097959 Fix for unrecognized flags on du (#488)
--total --block-size="K" --dereference were all unrecognized flags for this script. I have updated them to the correct flags to run correctly.

Co-authored-by: Stefan Benten <mail@stefan-benten.de>
2023-01-04 12:57:40 +01:00
Andrea Spacca
ca798ff6f6 Update README.md
add `-v` on sample command, print both x-url-delete and upload link in x-url-delete example
2022-09-22 11:58:07 +02:00
am97
31520b1afd Add /tmp dir to Docker image (#506)
fixes #462

Co-authored-by: Andrés Maldonado <maldonado@codelutin.com>
2022-08-31 08:07:29 +02:00
Andrea Spacca
3588502c50 bump transfer.sh-web dep (#504) 2022-08-24 11:01:54 +02:00
Farhan Khursheed
31ad4e01e1 Fixed improper implementation of content type (#501)
* Fixed improper implementation of content type

Add text/plain content type if metadata.ContentType is unable to determine the content type of the file

* Update server/handlers.go

Co-authored-by: Farhan Khursheed <41877347+blind-intruder@users.noreply.github.com>

* Update handlers.go

let's trim space

* Update handlers.go

Co-authored-by: Andrea Spacca <andrea.spacca@gmail.com>
2022-08-20 15:29:52 +02:00
Andrea Spacca
343427d3b9 Update README.md
fixes #483
2022-07-31 09:53:20 +02:00
Stefan Benten
64c7759126 all: update gdrive client and various linting cleanups (#497) 2022-07-14 18:02:18 +02:00
Stefan Benten
21812d3efc server: reorganize storage layer into more clear subfolder (#496)
* server: reorganize storage layer into more clear subfolder

* cmd: fix version command
2022-07-13 06:26:42 +02:00
Andrea Spacca
35e794220b Issue 485: fix IDN unicode/punycode handling (#486) 2022-05-15 16:48:13 +02:00
Ramon Fischer
f06aef1c3e Example Zsh/Bash function for transfer.sh homepage (#478)
* correct typo

* adapt title

* update function, correct typo, adapt sample output

* adapt subheading

* improve readability, reuse variables, save delete token as separate variable

* improve readability

* put function into expandable detail section

Co-authored-by: Stefan Benten <mail@stefan-benten.de>
2022-04-20 15:40:49 +02:00
Stefan Benten
eeff2c88be .github/,go.mod,go.sum: Update storj dependencies, drop older go versions (#468)
* go.mod,go.sum: Update storj dependencies

* bump go version

* another bump to latest storj release

* set minimum go version in go.mod

Co-authored-by: Andrea Spacca <andrea.spacca@gmail.com>
2022-04-20 01:50:44 +02:00
Stefan Benten
92324798d5 server: adding no-store header (#476)
In order to prevent viewing content, which max-download rate has been reached,
we need to ensure the data is not stored locally in a browser cache.
To achieve this, we set the Cache-Control Setting to "no-store" according to:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control

fixes #470
2022-04-10 12:13:06 +02:00
Ramon Fischer
b30b296ac8 correct typo (#480) 2022-04-06 01:33:40 +02:00
jeanluc
bb0891cd7d Docker: Allow selection of (unprivileged) UID/GID at build time (#418)
* Docker: use custom non-root UID/GID (build-arg)
2022-04-03 14:55:15 +02:00
Andrea Spacca
9c31ceb2c5 Update README.md 2022-04-03 06:17:51 +02:00
Anirudh Haritas Murali
597554a59e Add X-Url-Delete-* headers to POST handler (#435)
* Add X-Url-Delete-* headers to POST handler

* Remove token from header

* Fix writing of headers

* Handle error returned by Write

* Update handlers.go

Co-authored-by: Andrea Spacca <andrea.spacca@gmail.com>
2022-03-02 13:26:00 +01:00
Stefan Benten
368431fb6b server/storage.go: Update storj dependencies and set user-agent (#467) 2022-02-09 12:37:11 +01:00
Andrea Spacca
e3bb49993c fix perform-clamav-prescan (#460) 2022-01-10 11:01:26 +01:00
Andrea Spacca
cff0a88bf3 Clamav prescan (#389) 2022-01-09 22:14:10 +01:00
22 changed files with 2304 additions and 1166 deletions

View File

@@ -34,9 +34,11 @@ jobs:
fi
TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
TAGS_NOROOT="--tag ${DOCKER_IMAGE}:${VERSION}-noroot"
if [ $VERSION = edge -o $VERSION = nightly ]; then
TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
TAGS_NOROOT="$TAGS_NOROOT --tag ${DOCKER_IMAGE}:latest-noroot"
fi
echo ::set-output name=docker_image::${DOCKER_IMAGE}
@@ -46,6 +48,12 @@ jobs:
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
--build-arg VCS_REF=${GITHUB_SHA::8} \
${TAGS} .
echo ::set-output name=buildx_args_noroot::--platform ${DOCKER_PLATFORMS} \
--build-arg VERSION=${VERSION} \
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
--build-arg VCS_REF=${GITHUB_SHA::8} \
--build-arg RUNAS=noroot \
${TAGS_NOROOT} .
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
@@ -64,6 +72,7 @@ jobs:
name: Docker Buildx (build)
run: |
docker buildx build --no-cache --pull --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }}
docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args_noroot }}
-
name: Docker Login
if: success() && github.event_name != 'pull_request'
@@ -77,11 +86,13 @@ jobs:
if: success() && github.event_name != 'pull_request'
run: |
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args_noroot }}
-
name: Docker Check Manifest
if: always() && github.event_name != 'pull_request'
run: |
docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}
docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}-noroot
-
name: Clear
if: always() && github.event_name != 'pull_request'

View File

@@ -107,7 +107,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ^1.17
go-version: ^1.18
- name: Get project dependencies
run: go mod download

View File

@@ -13,11 +13,10 @@ jobs:
fail-fast: false
matrix:
go_version:
- 1.13.x
- 1.14.x
- 1.15.x
- 1.16.x
- 1.17.x
- 1.18.X
name: Test with ${{ matrix.go_version }}
steps:
- uses: actions/checkout@v2
@@ -36,7 +35,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-go@v1
with:
go-version: 1.17
go-version: 1.18
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:

View File

@@ -21,5 +21,5 @@ This code of conduct applies both within project spaces and in public spaces whe
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant] (https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html

View File

@@ -14,12 +14,28 @@ ENV GO111MODULE=on
# build & install server
RUN CGO_ENABLED=0 go build -tags netgo -ldflags "-X github.com/dutchcoders/transfer.sh/cmd.Version=$(git describe --tags) -a -s -w -extldflags '-static'" -o /go/bin/transfersh
ARG PUID=5000 \
PGID=5000 \
RUNAS
RUN mkdir -p /tmp/useradd /tmp/empty && \
if [ ! -z "$RUNAS" ]; then \
echo "${RUNAS}:x:${PUID}:${PGID}::/nonexistent:/sbin/nologin" >> /tmp/useradd/passwd && \
echo "${RUNAS}:!:::::::" >> /tmp/useradd/shadow && \
echo "${RUNAS}:x:${PGID}:" >> /tmp/useradd/group && \
echo "${RUNAS}:!::" >> /tmp/useradd/groupshadow; else touch /tmp/useradd/unused; fi
FROM scratch AS final
LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>"
ARG RUNAS
COPY --from=build /go/bin/transfersh /go/bin/transfersh
COPY --from=build /tmp/empty /tmp
COPY --from=build /tmp/useradd/* /etc/
COPY --from=build --chown=${RUNAS} /go/bin/transfersh /go/bin/transfersh
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
USER ${RUNAS}
ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"]
EXPOSE 8080

5
Makefile Normal file
View File

@@ -0,0 +1,5 @@
.PHONY: lint
lint:
golangci-lint run --out-format=github-actions --config .golangci.yml

193
README.md
View File

@@ -12,7 +12,7 @@ The service at transfersh.com is of unknown origin and reported as cloud malware
### Upload:
```bash
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt
$ curl -v --upload-file ./hello.txt https://transfer.sh/hello.txt
```
### Encrypt & Upload:
@@ -53,8 +53,9 @@ $ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "Max-Days: 1"
The URL used to request the deletion of a file and returned as a response header.
```bash
curl -sD - --upload-file ./hello https://transfer.sh/hello.txt | grep 'X-Url-Delete'
X-Url-Delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
curl -sD - --upload-file ./hello.txt https://transfer.sh/hello.txt | grep -i -E 'transfer\.sh|x-url-delete'
x-url-delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
https://transfer.sh/hello.txt/BAYh0/hello.txt
```
## Examples
@@ -111,6 +112,7 @@ lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated
log | path to log file| | LOG |
cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS |
clamav-host | host for clamav feature | | CLAMAV_HOST |
perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |
rate-limit | request per minute | | RATE_LIMIT |
max-upload-size | max upload size in kilobytes | | MAX_UPLOAD_SIZE |
purge-days | number of days after the uploads are purged automatically | | PURGE_DAYS |
@@ -139,12 +141,31 @@ $ go build -o transfersh main.go
## Docker
For easy deployment, we've created a Docker container.
For easy deployment, we've created an official Docker container. There are two variants, differing only by which user runs the process.
The default one will run as `root`:
```bash
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/
```
The one tagged with the suffix `-noroot` will use `5000` as both UID and GID:
```bash
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest-noroot --provider local --basedir /tmp/
```
### Building the Container
You can also build the container yourself. This allows you to choose which UID/GID will be used, e.g. when using NFS mounts:
```bash
# Build arguments:
# * RUNAS: If empty, the container will run as root.
# Set this to anything to enable UID/GID selection.
# * PUID: UID of the process. Needs RUNAS != "". Defaults to 5000.
# * PGID: GID of the process. Needs RUNAS != "". Defaults to 5000.
docker build -t transfer.sh-noroot --build-arg RUNAS=doesntmatter --build-arg PUID=1337 --build-arg PGID=1338 .
```
## S3 Usage
For the usage with a AWS S3 Bucket, you just need to specify the following options:
@@ -203,6 +224,170 @@ You need to create an OAuth Client id from console.cloud.google.com, download th
```go run main.go --provider gdrive --basedir /tmp/ --gdrive-client-json-filepath /[credential_dir] --gdrive-local-config-path [directory_to_save_config] ```
## Shell functions
### Bash and zsh (multiple files uploaded as zip archive)
##### Add this to .bashrc or .zshrc or its equivalent
```bash
transfer(){ if [ $# -eq 0 ];then echo "No arguments specified.\nUsage:\n transfer <file|directory>\n ... | transfer <file_name>">&2;return 1;fi;if tty -s;then file="$1";file_name=$(basename "$file");if [ ! -e "$file" ];then echo "$file: No such file or directory">&2;return 1;fi;if [ -d "$file" ];then file_name="$file_name.zip" ,;(cd "$file"&&zip -r -q - .)|curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null,;else cat "$file"|curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null;fi;else file_name=$1;curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null;fi;}
```
#### Now you can use transfer function
```
$ transfer hello.txt
```
### Bash and zsh (with delete url, delete token output and prompt before uploading)
##### Add this to .bashrc or .zshrc or its equivalent
<details><summary>Expand</summary><p>
```bash
transfer()
{
local file
declare -a file_array
file_array=("${@}")
if [[ "${file_array[@]}" == "" || "${1}" == "--help" || "${1}" == "-h" ]]
then
echo "${0} - Upload arbitrary files to \"transfer.sh\"."
echo ""
echo "Usage: ${0} [options] [<file>]..."
echo ""
echo "OPTIONS:"
echo " -h, --help"
echo " show this message"
echo ""
echo "EXAMPLES:"
echo " Upload a single file from the current working directory:"
echo " ${0} \"image.img\""
echo ""
echo " Upload multiple files from the current working directory:"
echo " ${0} \"image.img\" \"image2.img\""
echo ""
echo " Upload a file from a different directory:"
echo " ${0} \"/tmp/some_file\""
echo ""
echo " Upload all files from the current working directory. Be aware of the webserver's rate limiting!:"
echo " ${0} *"
echo ""
echo " Upload a single file from the current working directory and filter out the delete token and download link:"
echo " ${0} \"image.img\" | awk --field-separator=\": \" '/Delete token:/ { print \$2 } /Download link:/ { print \$2 }'"
echo ""
echo " Show help text from \"transfer.sh\":"
echo " curl --request GET \"https://transfer.sh\""
return 0
else
for file in "${file_array[@]}"
do
if [[ ! -f "${file}" ]]
then
echo -e "\e[01;31m'${file}' could not be found or is not a file.\e[0m" >&2
return 1
fi
done
unset file
fi
local upload_files
local curl_output
local awk_output
du -c -k -L "${file_array[@]}" >&2
# be compatible with "bash"
if [[ "${ZSH_NAME}" == "zsh" ]]
then
read $'upload_files?\e[01;31mDo you really want to upload the above files ('"${#file_array[@]}"$') to "transfer.sh"? (Y/n): \e[0m'
elif [[ "${BASH}" == *"bash"* ]]
then
read -p $'\e[01;31mDo you really want to upload the above files ('"${#file_array[@]}"$') to "transfer.sh"? (Y/n): \e[0m' upload_files
fi
case "${upload_files:-y}" in
"y"|"Y")
# for the sake of the progress bar, execute "curl" for each file.
# the parameters "--include" and "--form" will suppress the progress bar.
for file in "${file_array[@]}"
do
# show delete link and filter out the delete token from the response header after upload.
# it is important to save "curl's" "stdout" via a subshell to a variable or redirect it to another command,
# which just redirects to "stdout" in order to have a sane output afterwards.
# the progress bar is redirected to "stderr" and is only displayed,
# if "stdout" is redirected to something; e.g. ">/dev/null", "tee /dev/null" or "| <some_command>".
# the response header is redirected to "stdout", so redirecting "stdout" to "/dev/null" does not make any sense.
# redirecting "curl's" "stderr" to "stdout" ("2>&1") will suppress the progress bar.
curl_output=$(curl --request PUT --progress-bar --dump-header - --upload-file "${file}" "https://transfer.sh/")
awk_output=$(awk \
'gsub("\r", "", $0) && tolower($1) ~ /x-url-delete/ \
{
delete_link=$2;
print "Delete command: curl --request DELETE " "\""delete_link"\"";
gsub(".*/", "", delete_link);
delete_token=delete_link;
print "Delete token: " delete_token;
}
END{
print "Download link: " $0;
}' <<< "${curl_output}")
# return the results via "stdout", "awk" does not do this for some reason.
echo -e "${awk_output}\n"
# avoid rate limiting as much as possible; nginx: too many requests.
if (( ${#file_array[@]} > 4 ))
then
sleep 5
fi
done
;;
"n"|"N")
return 1
;;
*)
echo -e "\e[01;31mWrong input: '${upload_files}'.\e[0m" >&2
return 1
esac
}
```
</p></details>
#### Sample output
```bash
$ ls -lh
total 20M
-rw-r--r-- 1 <some_username> <some_username> 10M Apr 4 21:08 image.img
-rw-r--r-- 1 <some_username> <some_username> 10M Apr 4 21:08 image2.img
$ transfer image*
10240K image2.img
10240K image.img
20480K total
Do you really want to upload the above files (2) to "transfer.sh"? (Y/n):
######################################################################################################################################################################################################################################## 100.0%
Delete command: curl --request DELETE "https://transfer.sh/wJw9pz/image2.img/mSctGx7pYCId"
Delete token: mSctGx7pYCId
Download link: https://transfer.sh/wJw9pz/image2.img
######################################################################################################################################################################################################################################## 100.0%
Delete command: curl --request DELETE "https://transfer.sh/ljJc5I/image.img/nw7qaoiKUwCU"
Delete token: nw7qaoiKUwCU
Download link: https://transfer.sh/ljJc5I/image.img
$ transfer "image.img" | awk --field-separator=": " '/Delete token:/ { print $2 } /Download link:/ { print $2 }'
10240K image.img
10240K total
Do you really want to upload the above files (1) to "transfer.sh"? (Y/n):
######################################################################################################################################################################################################################################## 100.0%
tauN5dE3fWJe
https://transfer.sh/MYkuqn/image.img
```
## Contributions
Contributions are welcome.

View File

@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"github.com/dutchcoders/transfer.sh/server/storage"
"log"
"os"
"strings"
@@ -240,6 +241,11 @@ var globalFlags = []cli.Flag{
Value: "",
EnvVar: "CLAMAV_HOST",
},
cli.BoolFlag{
Name: "perform-clamav-prescan",
Usage: "perform-clamav-prescan",
EnvVar: "PERFORM_CLAMAV_PRESCAN",
},
cli.StringFlag{
Name: "virustotal-key",
Usage: "virustotal-key",
@@ -294,8 +300,8 @@ type Cmd struct {
*cli.App
}
func versionAction(c *cli.Context) {
fmt.Println(color.YellowString(fmt.Sprintf("transfer.sh %s: Easy file sharing from the command line", Version)))
func versionCommand(_ *cli.Context) {
fmt.Println(color.YellowString("transfer.sh %s: Easy file sharing from the command line", Version))
}
// New is the factory for transfer.sh
@@ -313,7 +319,7 @@ func New() *Cmd {
app.Commands = []cli.Command{
{
Name: "version",
Action: versionAction,
Action: versionCommand,
},
}
@@ -322,7 +328,7 @@ func New() *Cmd {
}
app.Action = func(c *cli.Context) {
options := []server.OptionFn{}
var options []server.OptionFn
if v := c.String("listener"); v != "" {
options = append(options, server.Listener(v))
}
@@ -388,6 +394,14 @@ func New() *Cmd {
options = append(options, server.ClamavHost(v))
}
if v := c.Bool("perform-clamav-prescan"); v {
if c.String("clamav-host") == "" {
panic("clamav-host not set")
}
options = append(options, server.PerformClamavPrescan(v))
}
if v := c.Int64("max-upload-size"); v > 0 {
options = append(options, server.MaxUploadSize(v))
}
@@ -450,42 +464,42 @@ func New() *Cmd {
panic("secret-key not set.")
} else if bucket := c.String("bucket"); bucket == "" {
panic("bucket not set.")
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
} else if store, err := storage.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
panic(err)
} else {
options = append(options, server.UseStorage(storage))
options = append(options, server.UseStorage(store))
}
case "gdrive":
chunkSize := c.Int("gdrive-chunk-size")
chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024
if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
panic("client-json-filepath not set.")
panic("gdrive-client-json-filepath not set.")
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
panic("local-config-path not set.")
panic("gdrive-local-config-path not set.")
} else if basedir := c.String("basedir"); basedir == "" {
panic("basedir not set.")
} else if storage, err := server.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
} else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
panic(err)
} else {
options = append(options, server.UseStorage(storage))
options = append(options, server.UseStorage(store))
}
case "storj":
if access := c.String("storj-access"); access == "" {
panic("storj-access not set.")
} else if bucket := c.String("storj-bucket"); bucket == "" {
panic("storj-bucket not set.")
} else if storage, err := server.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
} else if store, err := storage.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
panic(err)
} else {
options = append(options, server.UseStorage(storage))
options = append(options, server.UseStorage(store))
}
case "local":
if v := c.String("basedir"); v == "" {
panic("basedir not set.")
} else if storage, err := server.NewLocalStorage(v, logger); err != nil {
} else if store, err := storage.NewLocalStorage(v, logger); err != nil {
panic(err)
} else {
options = append(options, server.UseStorage(storage))
options = append(options, server.UseStorage(store))
}
default:
panic("Provider not set or invalid.")

26
go.mod
View File

@@ -1,20 +1,23 @@
module github.com/dutchcoders/transfer.sh
go 1.13
go 1.15
require (
cloud.google.com/go v0.77.0 // indirect
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2
github.com/aws/aws-sdk-go v1.37.14
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329
github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3
github.com/dutchcoders/transfer.sh-web v0.0.0-20220824020025-7240e75c3bb8
github.com/elazarl/go-bindata-assetfs v1.0.1
github.com/fatih/color v1.10.0
github.com/garyburd/redigo v1.6.2 // indirect
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.2 // indirect
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/gorilla/securecookie v1.1.1 // indirect
@@ -24,13 +27,14 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
github.com/urfave/cli v1.22.5
go.opencensus.io v0.22.6 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
google.golang.org/api v0.40.0
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 // indirect
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
golang.org/x/net v0.6.0 // indirect
golang.org/x/oauth2 v0.5.0
google.golang.org/api v0.109.0
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
storj.io/common v0.0.0-20211102144601-401a79f0706a
storj.io/uplink v1.7.1
storj.io/common v0.0.0-20220405183405-ffdc3ab808c6
storj.io/uplink v1.8.2
)

772
go.sum

File diff suppressed because it is too large Load Diff

View File

@@ -27,18 +27,19 @@ THE SOFTWARE.
package server
import (
// _ "transfer.sh/app/handlers"
// _ "transfer.sh/app/utils"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
clamd "github.com/dutchcoders/go-clamd"
"github.com/dutchcoders/go-clamd"
"github.com/gorilla/mux"
)
const clamavScanStatusOK = "OK"
func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
@@ -49,23 +50,53 @@ func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
s.logger.Printf("Scanning %s %d %s", filename, contentLength, contentType)
reader := r.Body
c := clamd.NewClamd(s.ClamAVDaemonHost)
abort := make(chan bool)
defer close(abort)
response, err := c.ScanStream(reader, abort)
file, err := ioutil.TempFile(s.tempPath, "clamav-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, err = io.Copy(file, r.Body)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", status)))
}
func (s *Server) performScan(path string) (string, error) {
c := clamd.NewClamd(s.ClamAVDaemonHost)
responseCh := make(chan chan *clamd.ScanResult)
errCh := make(chan error)
go func(responseCh chan chan *clamd.ScanResult, errCh chan error) {
response, err := c.ScanFile(path)
if err != nil {
errCh <- err
return
}
responseCh <- response
}(responseCh, errCh)
select {
case s := <-response:
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", s.Status)))
case err := <-errCh:
return "", err
case response := <-responseCh:
st := <-response
return st.Status, nil
case <-time.After(time.Second * 60):
abort <- true
return "", errors.New("clamav scan timeout")
}
}

View File

@@ -37,7 +37,7 @@ import (
"errors"
"fmt"
"html"
html_template "html/template"
htmlTemplate "html/template"
"io"
"io/ioutil"
"mime"
@@ -50,14 +50,17 @@ import (
"strconv"
"strings"
"sync"
text_template "text/template"
textTemplate "text/template"
"time"
"github.com/dutchcoders/transfer.sh/server/storage"
web "github.com/dutchcoders/transfer.sh-web"
"github.com/gorilla/mux"
"github.com/microcosm-cc/bluemonday"
blackfriday "github.com/russross/blackfriday/v2"
"github.com/russross/blackfriday/v2"
"github.com/skip2/go-qrcode"
"golang.org/x/net/idna"
)
const getPathPart = "get"
@@ -71,24 +74,24 @@ func stripPrefix(path string) string {
return strings.Replace(path, web.Prefix+"/", "", -1)
}
func initTextTemplates() *text_template.Template {
templateMap := text_template.FuncMap{"format": formatNumber}
func initTextTemplates() *textTemplate.Template {
templateMap := textTemplate.FuncMap{"format": formatNumber}
// Templates with functions available to them
var templates = text_template.New("").Funcs(templateMap)
var templates = textTemplate.New("").Funcs(templateMap)
return templates
}
func initHTMLTemplates() *html_template.Template {
templateMap := html_template.FuncMap{"format": formatNumber}
func initHTMLTemplates() *htmlTemplate.Template {
templateMap := htmlTemplate.FuncMap{"format": formatNumber}
// Templates with functions available to them
var templates = html_template.New("").Funcs(templateMap)
var templates = htmlTemplate.New("").Funcs(templateMap)
return templates
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
func healthHandler(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("Approaching Neutral Zone, all systems normal and functioning."))
}
@@ -136,7 +139,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
}
var templatePath string
var content html_template.HTML
var content htmlTemplate.HTML
switch {
case strings.HasPrefix(contentType, "image/"):
@@ -149,7 +152,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
templatePath = "download.markdown.html"
var reader io.ReadCloser
if reader, _, err = s.storage.Get(r.Context(), token, filename); err != nil {
if reader, _, err = s.storage.Get(r.Context(), token, filename, nil); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@@ -164,9 +167,9 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(contentType, "text/x-markdown") || strings.HasPrefix(contentType, "text/markdown") {
unsafe := blackfriday.Run(data)
output := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
content = html_template.HTML(output)
content = htmlTemplate.HTML(output)
} else if strings.HasPrefix(contentType, "text/plain") {
content = html_template.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
content = htmlTemplate.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
} else {
templatePath = "download.sandbox.html"
}
@@ -193,7 +196,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
data := struct {
ContentType string
Content html_template.HTML
Content htmlTemplate.HTML
Filename string
URL string
URLGet string
@@ -280,7 +283,7 @@ func (s *Server) viewHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (s *Server) notFoundHandler(w http.ResponseWriter, r *http.Request) {
func (s *Server) notFoundHandler(w http.ResponseWriter, _ *http.Request) {
http.Error(w, http.StatusText(404), 404)
}
@@ -291,7 +294,7 @@ func sanitize(fileName string) string {
func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(_24K); nil != err {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
@@ -299,93 +302,103 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
for _, fheaders := range r.MultipartForm.File {
for _, fheader := range fheaders {
filename := sanitize(fheader.Filename)
contentType := mime.TypeByExtension(filepath.Ext(fheader.Filename))
responseBody := ""
for _, fHeaders := range r.MultipartForm.File {
for _, fHeader := range fHeaders {
filename := sanitize(fHeader.Filename)
contentType := mime.TypeByExtension(filepath.Ext(fHeader.Filename))
var f io.Reader
var err error
if f, err = fheader.Open(); err != nil {
if f, err = fHeader.Open(); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var b bytes.Buffer
file, err := ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
n, err := io.CopyN(&b, f, _24K+1)
if err != nil && err != io.EOF {
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var file *os.File
var reader io.Reader
if n > _24K {
file, err = ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Fatal(err)
}
n, err = io.Copy(file, io.MultiReader(&b, f))
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
reader, err = os.Open(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
} else {
reader = bytes.NewReader(b.Bytes())
n, err := io.Copy(file, f)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
contentLength := n
_, err = file.Seek(0, io.SeekStart)
if err != nil {
s.logger.Printf("%s", err.Error())
return
}
if s.maxUploadSize > 0 && contentLength > s.maxUploadSize {
s.logger.Print("Entity too large")
http.Error(w, http.StatusText(http.StatusRequestEntityTooLarge), http.StatusRequestEntityTooLarge)
return
}
if s.performClamavPrescan {
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
return
}
if status != clamavScanStatusOK {
s.logger.Printf("prescan positive: %s", status)
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
return
}
}
metadata := metadataForRequest(contentType, s.randomTokenLength, r)
buffer := &bytes.Buffer{}
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not encode metadata", 500)
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
return
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not save metadata", 500)
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
return
}
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
s.logger.Printf("Backend storage error: %s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
filename = url.PathEscape(filename)
relativeURL, _ := url.Parse(path.Join(s.proxyPath, token, filename))
_, _ = w.Write([]byte(getURL(r, s.proxyPort).ResolveReference(relativeURL).String()))
deleteURL, _ := url.Parse(path.Join(s.proxyPath, token, filename, metadata.DeletionToken))
w.Header().Add("X-Url-Delete", resolveURL(r, deleteURL, s.proxyPort))
responseBody += fmt.Sprintln(getURL(r, s.proxyPort).ResolveReference(relativeURL).String())
}
}
_, err := w.Write([]byte(responseBody))
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
func (s *Server) cleanTmpFile(f *os.File) {
@@ -448,56 +461,35 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
contentLength := r.ContentLength
var reader io.Reader
defer storage.CloseCheck(r.Body)
reader = r.Body
file, err := ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer CloseCheck(r.Body.Close)
// queue file to disk, because s3 needs content length
// and clamav prescan scans a file
n, err := io.Copy(file, r.Body)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
if contentLength == -1 {
// queue file to disk, because s3 needs content length
var err error
return
}
f := reader
_, err = file.Seek(0, io.SeekStart)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Cannot reset cache file", http.StatusInternalServerError)
var b bytes.Buffer
n, err := io.CopyN(&b, f, _24K+1)
if err != nil && err != io.EOF {
s.logger.Printf("Error putting new file: %s", err.Error())
http.Error(w, err.Error(), 500)
return
}
var file *os.File
if n > _24K {
file, err = ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
n, err = io.Copy(file, io.MultiReader(&b, f))
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
reader, err = os.Open(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
} else {
reader = bytes.NewReader(b.Bytes())
}
return
}
if contentLength < 1 {
contentLength = n
}
@@ -509,10 +501,25 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
if contentLength == 0 {
s.logger.Print("Empty content-length")
http.Error(w, "Could not upload empty file", 400)
http.Error(w, "Could not upload empty file", http.StatusBadRequest)
return
}
if s.performClamavPrescan {
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
return
}
if status != clamavScanStatusOK {
s.logger.Printf("prescan positive: %s", status)
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
return
}
}
contentType := mime.TypeByExtension(filepath.Ext(vars["filename"]))
token := token(s.randomTokenLength)
@@ -522,25 +529,23 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
buffer := &bytes.Buffer{}
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not encode metadata", 500)
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
return
} else if !metadata.MaxDate.IsZero() && time.Now().After(metadata.MaxDate) {
s.logger.Print("Invalid MaxDate")
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", 400)
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", http.StatusBadRequest)
return
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not save metadata", 500)
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
return
}
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
var err error
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
s.logger.Printf("Error putting new file: %s", err.Error())
http.Error(w, "Could not save file", 500)
http.Error(w, "Could not save file", http.StatusInternalServerError)
return
}
@@ -622,6 +627,14 @@ func getURL(r *http.Request, proxyPort string) *url.URL {
host = r.Host
port = ""
}
p := idna.New(idna.ValidateForRegistration())
var hostFromPunycode string
hostFromPunycode, err = p.ToUnicode(host)
if err == nil {
host = hostFromPunycode
}
if len(proxyPort) != 0 {
port = proxyPort
}
@@ -680,8 +693,8 @@ func (s *Server) checkMetadata(ctx context.Context, token, filename string, incr
var metadata metadata
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
defer CloseCheck(r.Close)
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
defer storage.CloseCheck(r)
if err != nil {
return metadata, err
@@ -716,8 +729,8 @@ func (s *Server) checkDeletionToken(ctx context.Context, deletionToken, token, f
var metadata metadata
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
defer CloseCheck(r.Close)
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
defer storage.CloseCheck(r)
if s.storage.IsNotExist(err) {
return errors.New("metadata doesn't exist")
@@ -766,7 +779,7 @@ func (s *Server) deleteHandler(w http.ResponseWriter, r *http.Request) {
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not delete file.", 500)
http.Error(w, "Could not delete file.", http.StatusInternalServerError)
return
}
}
@@ -779,8 +792,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
zipfilename := fmt.Sprintf("transfersh-%d.zip", uint16(time.Now().UnixNano()))
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", zipfilename))
w.Header().Set("Connection", "close")
commonHeader(w, zipfilename)
zw := zip.NewWriter(w)
@@ -795,8 +807,8 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
continue
}
reader, _, err := s.storage.Get(r.Context(), token, filename)
defer CloseCheck(reader.Close)
reader, _, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -805,7 +817,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@@ -820,20 +832,20 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(fw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
if err := zw.Close(); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@@ -846,14 +858,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
tarfilename := fmt.Sprintf("transfersh-%d.tar.gz", uint16(time.Now().UnixNano()))
w.Header().Set("Content-Type", "application/x-gzip")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", tarfilename))
w.Header().Set("Connection", "close")
commonHeader(w, tarfilename)
gw := gzip.NewWriter(w)
defer CloseCheck(gw.Close)
defer storage.CloseCheck(gw)
zw := tar.NewWriter(gw)
defer CloseCheck(zw.Close)
defer storage.CloseCheck(zw)
for _, key := range strings.Split(files, ",") {
key = resolveKey(key, s.proxyPath)
@@ -866,8 +877,8 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
continue
}
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
defer CloseCheck(reader.Close)
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -876,7 +887,7 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@@ -888,13 +899,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
err = zw.WriteHeader(header)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(zw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@@ -908,11 +919,10 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
tarfilename := fmt.Sprintf("transfersh-%d.tar", uint16(time.Now().UnixNano()))
w.Header().Set("Content-Type", "application/x-tar")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", tarfilename))
w.Header().Set("Connection", "close")
commonHeader(w, tarfilename)
zw := tar.NewWriter(w)
defer CloseCheck(zw.Close)
defer storage.CloseCheck(zw)
for _, key := range strings.Split(files, ",") {
key = resolveKey(key, s.proxyPath)
@@ -925,8 +935,8 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
continue
}
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
defer CloseCheck(reader.Close)
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
defer storage.CloseCheck(reader)
if err != nil {
if s.storage.IsNotExist(err) {
@@ -935,7 +945,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@@ -947,13 +957,13 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
err = zw.WriteHeader(header)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(zw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@@ -980,7 +990,7 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@@ -991,6 +1001,10 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Connection", "close")
w.Header().Set("X-Remaining-Downloads", remainingDownloads)
w.Header().Set("X-Remaining-Days", remainingDays)
if s.storage.IsRangeSupported() {
w.Header().Set("Accept-Ranges", "bytes")
}
}
func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
@@ -1008,23 +1022,49 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
return
}
var rng *storage.Range
if r.Header.Get("Range") != "" {
rng = storage.ParseRange(r.Header.Get("Range"))
}
contentType := metadata.ContentType
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
defer CloseCheck(reader.Close)
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, rng)
defer storage.CloseCheck(reader)
rdr := io.Reader(reader)
if s.storage.IsNotExist(err) {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
if rng != nil {
cr := rng.ContentRange()
if cr != "" {
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Range", cr)
if rng.Limit > 0 {
rdr = io.LimitReader(reader, int64(rng.Limit))
}
}
}
var disposition string
if action == "inline" {
disposition = "inline"
/*
metadata.ContentType is unable to determine the type of the content,
metadata.ContentType is unable to determine the type of the content,
metadata.ContentType is unable to determine the type of the content,
So add text/plain in this case to fix XSS related issues/
*/
if strings.TrimSpace(contentType) == "" {
contentType = "text/plain"
}
} else {
disposition = "attachment"
}
@@ -1035,41 +1075,31 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.FormatUint(contentLength, 10))
w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", disposition, filename))
w.Header().Set("Connection", "keep-alive")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Remaining-Downloads", remainingDownloads)
w.Header().Set("X-Remaining-Days", remainingDays)
if rng != nil && rng.ContentRange() != "" {
w.WriteHeader(http.StatusPartialContent)
}
if disposition == "inline" && canContainsXSS(contentType) {
reader = ioutil.NopCloser(bluemonday.UGCPolicy().SanitizeReader(reader))
}
if w.Header().Get("Range") != "" || strings.HasPrefix(metadata.ContentType, "video") || strings.HasPrefix(metadata.ContentType, "audio") {
file, err := ioutil.TempFile(s.tempPath, "range-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
return
}
_, err = io.Copy(file, reader)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
return
}
http.ServeContent(w, r, filename, time.Now(), file)
return
}
if _, err = io.Copy(w, reader); err != nil {
if _, err = io.Copy(w, rdr); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
}
func commonHeader(w http.ResponseWriter, filename string) {
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
w.Header().Set("Connection", "close")
w.Header().Set("Cache-Control", "no-store")
}
// RedirectHandler handles redirect
func (s *Server) RedirectHandler(h http.Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
@@ -1136,12 +1166,12 @@ func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
username, password, authOK := r.BasicAuth()
if !authOK {
http.Error(w, "Not authorized", 401)
http.Error(w, "Not authorized", http.StatusUnauthorized)
return
}
if username != s.AuthUser || password != s.AuthPass {
http.Error(w, "Not authorized", 401)
http.Error(w, "Not authorized", http.StatusUnauthorized)
return
}

View File

@@ -88,10 +88,10 @@ func (f *ipFilter) BlockIP(ip string) bool {
}
func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
//check if has subnet
if ip, net, err := net.ParseCIDR(str); err == nil {
//check if provided string describes a subnet
if ip, network, err := net.ParseCIDR(str); err == nil {
// containing only one ip?
if n, total := net.Mask.Size(); n == total {
if n, total := network.Mask.Size(); n == total {
f.mut.Lock()
f.ips[ip.String()] = allowed
f.mut.Unlock()
@@ -110,7 +110,7 @@ func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
if !found {
f.subnets = append(f.subnets, &subnet{
str: str,
ipnet: net,
ipnet: network,
allowed: allowed,
})
}
@@ -182,7 +182,7 @@ func (f *ipFilter) NetBlocked(ip net.IP) bool {
return !f.NetAllowed(ip)
}
//WrapIPFilter the provided handler with simple IP blocking middleware
//Wrap the provided handler with simple IP blocking middleware
//using this IP filter and its configuration
func (f *ipFilter) Wrap(next http.Handler) http.Handler {
return &ipFilterMiddleware{ipFilter: f, next: next}

View File

@@ -25,39 +25,35 @@ THE SOFTWARE.
package server
import (
crypto_rand "crypto/rand"
"context"
cryptoRand "crypto/rand"
"crypto/tls"
"encoding/binary"
"errors"
gorillaHandlers "github.com/gorilla/handlers"
"log"
"math/rand"
"mime"
"net/http"
_ "net/http/pprof"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
context "golang.org/x/net/context"
"github.com/PuerkitoBio/ghost/handlers"
"github.com/VojtechVitek/ratelimit"
"github.com/VojtechVitek/ratelimit/memory"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/gorilla/mux"
// import pprof
_ "net/http/pprof"
"crypto/tls"
"golang.org/x/crypto/acme/autocert"
web "github.com/dutchcoders/transfer.sh-web"
"github.com/dutchcoders/transfer.sh/server/storage"
assetfs "github.com/elazarl/go-bindata-assetfs"
autocert "golang.org/x/crypto/acme/autocert"
"path/filepath"
)
// parse request with maximum memory of _24Kilobits
@@ -76,6 +72,13 @@ func ClamavHost(s string) OptionFn {
}
}
// PerformClamavPrescan enables clamav prescan on upload
func PerformClamavPrescan(b bool) OptionFn {
return func(srvr *Server) {
srvr.performClamavPrescan = b
}
}
// VirustotalKey sets virus total key
func VirustotalKey(s string) OptionFn {
return func(srvr *Server) {
@@ -241,7 +244,7 @@ func EnableProfiler() OptionFn {
}
// UseStorage set storage to use
func UseStorage(s Storage) OptionFn {
func UseStorage(s storage.Storage) OptionFn {
return func(srvr *Server) {
srvr.storage = s
}
@@ -330,7 +333,7 @@ type Server struct {
purgeDays time.Duration
purgeInterval time.Duration
storage Storage
storage storage.Storage
forceHTTPS bool
@@ -338,8 +341,9 @@ type Server struct {
ipFilterOptions *IPFilterOptions
VirusTotalKey string
ClamAVDaemonHost string
VirusTotalKey string
ClamAVDaemonHost string
performClamavPrescan bool
tempPath string
@@ -377,7 +381,7 @@ func New(options ...OptionFn) (*Server, error) {
func init() {
var seedBytes [8]byte
if _, err := crypto_rand.Read(seedBytes[:]); err != nil {
if _, err := cryptoRand.Read(seedBytes[:]); err != nil {
panic("cannot obtain cryptographically secure seed")
}
rand.Seed(int64(binary.LittleEndian.Uint64(seedBytes[:])))
@@ -424,13 +428,17 @@ func (s *Server) Run() {
s.logger.Panicf("Unable to parse: path=%s, err=%s", path, err)
}
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse html template", err)
if strings.HasSuffix(path, ".html") {
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse html template", err)
}
}
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse text template", err)
if strings.HasSuffix(path, ".txt") {
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse text template", err)
}
}
}
}
@@ -467,7 +475,7 @@ func (s *Server) Run() {
return false
}
match = (r.Referer() == "")
match = r.Referer() == ""
u, err := url.Parse(r.Referer())
if err != nil {
@@ -529,32 +537,34 @@ func (s *Server) Run() {
)
if !s.TLSListenerOnly {
srvr := &http.Server{
Addr: s.ListenerString,
Handler: h,
}
listening = true
s.logger.Printf("listening on port: %v\n", s.ListenerString)
s.logger.Printf("starting to listen on: %v\n", s.ListenerString)
go func() {
_ = srvr.ListenAndServe()
srvr := &http.Server{
Addr: s.ListenerString,
Handler: h,
}
if err := srvr.ListenAndServe(); err != nil {
s.logger.Fatal(err)
}
}()
}
if s.TLSListenerString != "" {
listening = true
s.logger.Printf("listening on port: %v\n", s.TLSListenerString)
s.logger.Printf("starting to listen for TLS on: %v\n", s.TLSListenerString)
go func() {
s := &http.Server{
srvr := &http.Server{
Addr: s.TLSListenerString,
Handler: h,
TLSConfig: s.tlsConfig,
}
if err := s.ListenAndServeTLS("", ""); err != nil {
panic(err)
if err := srvr.ListenAndServeTLS("", ""); err != nil {
s.logger.Fatal(err)
}
}()
}

View File

@@ -1,808 +0,0 @@
package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"storj.io/common/fpath"
"storj.io/common/storj"
"storj.io/uplink"
)
// Storage is the interface for storage operation
type Storage interface {
// Get retrieves a file from storage
Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error)
// Head retrieves content length of a file from storage
Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
// Put saves a file on storage
Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
// Delete removes a file from storage
Delete(ctx context.Context, token string, filename string) error
// IsNotExist indicates if a file doesn't exist on storage
IsNotExist(err error) bool
// Purge cleans up the storage
Purge(ctx context.Context, days time.Duration) error
// Type returns the storage type
Type() string
}
// LocalStorage is a local storage
type LocalStorage struct {
Storage
basedir string
logger *log.Logger
}
// NewLocalStorage is the factory for LocalStorage
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
return &LocalStorage{basedir: basedir, logger: logger}, nil
}
// Type returns the storage type
func (s *LocalStorage) Type() string {
return "local"
}
// Head retrieves content length of a file from storage
func (s *LocalStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
path := filepath.Join(s.basedir, token, filename)
var fi os.FileInfo
if fi, err = os.Lstat(path); err != nil {
return
}
contentLength = uint64(fi.Size())
return
}
// Get retrieves a file from storage
func (s *LocalStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
path := filepath.Join(s.basedir, token, filename)
// content type , content length
if reader, err = os.Open(path); err != nil {
return
}
var fi os.FileInfo
if fi, err = os.Lstat(path); err != nil {
return
}
contentLength = uint64(fi.Size())
return
}
// Delete removes a file from storage
func (s *LocalStorage) Delete(ctx context.Context, token string, filename string) (err error) {
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
_ = os.Remove(metadata)
path := filepath.Join(s.basedir, token, filename)
err = os.Remove(path)
return
}
// Purge cleans up the storage
func (s *LocalStorage) Purge(ctx context.Context, days time.Duration) (err error) {
err = filepath.Walk(s.basedir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if info.ModTime().Before(time.Now().Add(-1 * days)) {
err = os.Remove(path)
return err
}
return nil
})
return
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *LocalStorage) IsNotExist(err error) bool {
if err == nil {
return false
}
return os.IsNotExist(err)
}
// Put saves a file on storage
func (s *LocalStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
var f io.WriteCloser
var err error
path := filepath.Join(s.basedir, token)
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
return err
}
f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f.Close)
if err != nil {
return err
}
if _, err = io.Copy(f, reader); err != nil {
return err
}
return nil
}
// S3Storage is a storage backed by AWS S3
type S3Storage struct {
Storage
bucket string
session *session.Session
s3 *s3.S3
logger *log.Logger
purgeDays time.Duration
noMultipart bool
}
// NewS3Storage is the factory for S3Storage
func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
return &S3Storage{
bucket: bucketName,
s3: s3.New(sess),
session: sess,
logger: logger,
noMultipart: disableMultipart,
purgeDays: time.Duration(purgeDays*24) * time.Hour,
}, nil
}
// Type returns the storage type
func (s *S3Storage) Type() string {
return "s3"
}
// Head retrieves content length of a file from storage
func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
key := fmt.Sprintf("%s/%s", token, filename)
headRequest := &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
// content type , content length
response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
if err != nil {
return
}
if response.ContentLength != nil {
contentLength = uint64(*response.ContentLength)
}
return
}
// Purge cleans up the storage
func (s *S3Storage) Purge(ctx context.Context, days time.Duration) (err error) {
// NOOP expiration is set at upload time
return nil
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *S3Storage) IsNotExist(err error) bool {
if err == nil {
return false
}
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchKey:
return true
}
}
return false
}
// Get retrieves a file from storage
func (s *S3Storage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
key := fmt.Sprintf("%s/%s", token, filename)
getRequest := &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
response, err := s.s3.GetObjectWithContext(ctx, getRequest)
if err != nil {
return
}
if response.ContentLength != nil {
contentLength = uint64(*response.ContentLength)
}
reader = response.Body
return
}
// Delete removes a file from storage
func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
deleteRequest := &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(metadata),
}
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
if err != nil {
return
}
key := fmt.Sprintf("%s/%s", token, filename)
deleteRequest = &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
return
}
// Put saves a file on storage
func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
key := fmt.Sprintf("%s/%s", token, filename)
s.logger.Printf("Uploading file %s to S3 Bucket", filename)
var concurrency int
if !s.noMultipart {
concurrency = 20
} else {
concurrency = 1
}
// Create an uploader with the session and custom options
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
u.Concurrency = concurrency // default is 5
u.LeavePartsOnError = false
})
var expire *time.Time
if s.purgeDays.Hours() > 0 {
expire = aws.Time(time.Now().Add(s.purgeDays))
}
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: reader,
Expires: expire,
})
return
}
// GDrive is a storage backed by GDrive
type GDrive struct {
service *drive.Service
rootID string
basedir string
localConfigPath string
chunkSize int
logger *log.Logger
}
// NewGDriveStorage is the factory for GDrive
func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
b, err := ioutil.ReadFile(clientJSONFilepath)
if err != nil {
return nil, err
}
// If modifying these scopes, delete your previously saved client_secret.json.
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
// ToDo: Upgrade deprecated version
srv, err := drive.New(getGDriveClient(context.TODO(), config, localConfigPath, logger)) // nolint: staticcheck
if err != nil {
return nil, err
}
chunkSize = chunkSize * 1024 * 1024
storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
err = storage.setupRoot()
if err != nil {
return nil, err
}
return storage, nil
}
const gdriveRootConfigFile = "root_id.conf"
const gdriveTokenJSONFile = "token.json"
const gdriveDirectoryMimeType = "application/vnd.google-apps.folder"
func (s *GDrive) setupRoot() error {
rootFileConfig := filepath.Join(s.localConfigPath, gdriveRootConfigFile)
rootID, err := ioutil.ReadFile(rootFileConfig)
if err != nil && !os.IsNotExist(err) {
return err
}
if string(rootID) != "" {
s.rootID = string(rootID)
return nil
}
dir := &drive.File{
Name: s.basedir,
MimeType: gdriveDirectoryMimeType,
}
di, err := s.service.Files.Create(dir).Fields("id").Do()
if err != nil {
return err
}
s.rootID = di.Id
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
if err != nil {
return err
}
return nil
}
func (s *GDrive) hasChecksum(f *drive.File) bool {
return f.Md5Checksum != ""
}
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
}
func (s *GDrive) findID(filename string, token string) (string, error) {
filename = strings.Replace(filename, `'`, `\'`, -1)
filename = strings.Replace(filename, `"`, `\"`, -1)
fileID, tokenID, nextPageToken := "", "", ""
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gdriveDirectoryMimeType)
l, err := s.list(nextPageToken, q)
if err != nil {
return "", err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
tokenID = fi.Id
break
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return "", err
}
}
if filename == "" {
return tokenID, nil
} else if tokenID == "" {
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
}
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gdriveDirectoryMimeType)
l, err = s.list(nextPageToken, q)
if err != nil {
return "", err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
fileID = fi.Id
break
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return "", err
}
}
if fileID == "" {
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
}
return fileID, nil
}
// Type returns the storage type
func (s *GDrive) Type() string {
return "gdrive"
}
// Head retrieves content length of a file from storage
func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
var fi *drive.File
if fi, err = s.service.Files.Get(fileID).Fields("size").Do(); err != nil {
return
}
contentLength = uint64(fi.Size)
return
}
// Get retrieves a file from storage
func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
var fi *drive.File
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
if err != nil {
return
}
if !s.hasChecksum(fi) {
err = fmt.Errorf("cannot find file %s/%s", token, filename)
return
}
contentLength = uint64(fi.Size)
var res *http.Response
res, err = s.service.Files.Get(fileID).Context(ctx).Download()
if err != nil {
return
}
reader = res.Body
return
}
// Delete removes a file from storage
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
_ = s.service.Files.Delete(metadata).Do()
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
err = s.service.Files.Delete(fileID).Do()
return
}
// Purge cleans up the storage
func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
nextPageToken := ""
expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gdriveDirectoryMimeType)
l, err := s.list(nextPageToken, q)
if err != nil {
return err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
err = s.service.Files.Delete(fi.Id).Do()
if err != nil {
return
}
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return
}
}
return
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *GDrive) IsNotExist(err error) bool {
if err == nil {
return false
}
if e, ok := err.(*googleapi.Error); ok {
return e.Code == http.StatusNotFound
}
return false
}
// Put saves a file on storage
func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
dirID, err := s.findID("", token)
if err != nil {
return err
}
if dirID == "" {
dir := &drive.File{
Name: token,
Parents: []string{s.rootID},
MimeType: gdriveDirectoryMimeType,
Size: int64(contentLength),
}
di, err := s.service.Files.Create(dir).Fields("id").Do()
if err != nil {
return err
}
dirID = di.Id
}
// Instantiate empty drive file
dst := &drive.File{
Name: filename,
Parents: []string{dirID},
MimeType: contentType,
}
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
if err != nil {
return err
}
return nil
}
// Retrieve a token, saves the token, then returns the generated client.
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
tokenFile := filepath.Join(localConfigPath, gdriveTokenJSONFile)
tok, err := gDriveTokenFromFile(tokenFile)
if err != nil {
tok = getGDriveTokenFromWeb(ctx, config, logger)
saveGDriveToken(tokenFile, tok, logger)
}
return config.Client(ctx, tok)
}
// Request a token from the web, then returns the retrieved token.
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
var authCode string
if _, err := fmt.Scan(&authCode); err != nil {
logger.Fatalf("Unable to read authorization code %v", err)
}
tok, err := config.Exchange(ctx, authCode)
if err != nil {
logger.Fatalf("Unable to retrieve token from web %v", err)
}
return tok
}
// Retrieves a token from a local file.
func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
defer CloseCheck(f.Close)
if err != nil {
return nil, err
}
tok := &oauth2.Token{}
err = json.NewDecoder(f).Decode(tok)
return tok, err
}
// Saves a token to a file path.
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
logger.Printf("Saving credential file to: %s\n", path)
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f.Close)
if err != nil {
logger.Fatalf("Unable to cache oauth token: %v", err)
}
err = json.NewEncoder(f).Encode(token)
if err != nil {
logger.Fatalf("Unable to encode oauth token: %v", err)
}
}
// StorjStorage is a storage backed by Storj
type StorjStorage struct {
Storage
project *uplink.Project
bucket *uplink.Bucket
purgeDays time.Duration
logger *log.Logger
}
// NewStorjStorage is the factory for StorjStorage
func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
var instance StorjStorage
var err error
pCtx := context.TODO()
ctx := fpath.WithTempData(pCtx, "", true)
parsedAccess, err := uplink.ParseAccess(access)
if err != nil {
return nil, err
}
instance.project, err = uplink.OpenProject(ctx, parsedAccess)
if err != nil {
return nil, err
}
instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
if err != nil {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = instance.project.Close()
return nil, err
}
instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
instance.logger = logger
return &instance, nil
}
// Type returns the storage type
func (s *StorjStorage) Type() string {
return "storj"
}
// Head retrieves content length of a file from storage
func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
key := storj.JoinPaths(token, filename)
obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
if err != nil {
return 0, err
}
contentLength = uint64(obj.System.ContentLength)
return
}
// Get retrieves a file from storage
func (s *StorjStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Getting file %s from Storj Bucket", filename)
download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, nil)
if err != nil {
return nil, 0, err
}
contentLength = uint64(download.Info().System.ContentLength)
reader = download
return
}
// Delete removes a file from storage
func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Deleting file %s from Storj Bucket", filename)
_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
return
}
// Purge cleans up the storage
func (s *StorjStorage) Purge(ctx context.Context, days time.Duration) (err error) {
// NOOP expiration is set at upload time
return nil
}
// Put saves a file on storage
func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Uploading file %s to Storj Bucket", filename)
var uploadOptions *uplink.UploadOptions
if s.purgeDays.Hours() > 0 {
uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
}
writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
if err != nil {
return err
}
n, err := io.Copy(writer, reader)
if err != nil || uint64(n) != contentLength {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = writer.Abort()
return err
}
err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
if err != nil {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = writer.Abort()
return err
}
err = writer.Commit()
return err
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *StorjStorage) IsNotExist(err error) bool {
return errors.Is(err, uplink.ErrObjectNotFound)
}

120
server/storage/common.go Normal file
View File

@@ -0,0 +1,120 @@
package storage
import (
"context"
"fmt"
"io"
"strconv"
"time"
"regexp"
)
type Range struct {
Start uint64
Limit uint64
contentRange string
}
// Range Reconstructs Range header and returns it
func (r *Range) Range() string {
if r.Limit > 0 {
return fmt.Sprintf("bytes=%d-%d", r.Start, r.Start+r.Limit-1)
} else {
return fmt.Sprintf("bytes=%d-", r.Start)
}
}
// AcceptLength Tries to accept given range
// returns newContentLength if range was satisfied, otherwise returns given contentLength
func (r *Range) AcceptLength(contentLength uint64) (newContentLength uint64) {
newContentLength = contentLength
if r.Limit == 0 {
r.Limit = newContentLength - r.Start
}
if contentLength < r.Start {
return
}
if r.Limit > contentLength-r.Start {
return
}
r.contentRange = fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Limit-1, contentLength)
newContentLength = r.Limit
return
}
func (r *Range) SetContentRange(cr string) {
r.contentRange = cr
}
// Returns accepted Content-Range header. If range wasn't accepted empty string is returned
func (r *Range) ContentRange() string {
return r.contentRange
}
var rexp *regexp.Regexp = regexp.MustCompile(`^bytes=([0-9]+)-([0-9]*)$`)
// Parses HTTP Range header and returns struct on success
// only bytes=start-finish supported
func ParseRange(rng string) *Range {
if rng == "" {
return nil
}
matches := rexp.FindAllStringSubmatch(rng, -1)
if len(matches) != 1 || len(matches[0]) != 3 {
return nil
}
if len(matches[0][0]) != len(rng) || len(matches[0][1]) == 0 {
return nil
}
start, err := strconv.ParseUint(matches[0][1], 10, 64)
if err != nil {
return nil
}
if len(matches[0][2]) == 0 {
return &Range{Start: start, Limit: 0}
}
finish, err := strconv.ParseUint(matches[0][2], 10, 64)
if err != nil {
return nil
}
if finish < start || finish+1 < finish {
return nil
}
return &Range{Start: start, Limit: finish - start + 1}
}
// Storage is the interface for storage operation
type Storage interface {
// Get retrieves a file from storage
Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error)
// Head retrieves content length of a file from storage
Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
// Put saves a file on storage
Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
// Delete removes a file from storage
Delete(ctx context.Context, token string, filename string) error
// IsNotExist indicates if a file doesn't exist on storage
IsNotExist(err error) bool
// Purge cleans up the storage
Purge(ctx context.Context, days time.Duration) error
// Whether storage supports Get with Range header
IsRangeSupported() bool
// Type returns the storage type
Type() string
}
func CloseCheck(c io.Closer) {
if c == nil {
return
}
if err := c.Close(); err != nil {
fmt.Println("Received close error:", err)
}
}

394
server/storage/gdrive.go Normal file
View File

@@ -0,0 +1,394 @@
package storage
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
)
// GDrive is a storage backed by GDrive
type GDrive struct {
service *drive.Service
rootID string
basedir string
localConfigPath string
chunkSize int
logger *log.Logger
}
const gDriveRootConfigFile = "root_id.conf"
const gDriveTokenJSONFile = "token.json"
const gDriveDirectoryMimeType = "application/vnd.google-apps.folder"
// NewGDriveStorage is the factory for GDrive
func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
ctx := context.TODO()
b, err := ioutil.ReadFile(clientJSONFilepath)
if err != nil {
return nil, err
}
// If modifying these scopes, delete your previously saved client_secret.json.
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
httpClient := getGDriveClient(ctx, config, localConfigPath, logger)
srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
if err != nil {
return nil, err
}
storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
err = storage.setupRoot()
if err != nil {
return nil, err
}
return storage, nil
}
func (s *GDrive) setupRoot() error {
rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile)
rootID, err := ioutil.ReadFile(rootFileConfig)
if err != nil && !os.IsNotExist(err) {
return err
}
if string(rootID) != "" {
s.rootID = string(rootID)
return nil
}
dir := &drive.File{
Name: s.basedir,
MimeType: gDriveDirectoryMimeType,
}
di, err := s.service.Files.Create(dir).Fields("id").Do()
if err != nil {
return err
}
s.rootID = di.Id
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
if err != nil {
return err
}
return nil
}
func (s *GDrive) hasChecksum(f *drive.File) bool {
return f.Md5Checksum != ""
}
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
}
func (s *GDrive) findID(filename string, token string) (string, error) {
filename = strings.Replace(filename, `'`, `\'`, -1)
filename = strings.Replace(filename, `"`, `\"`, -1)
fileID, tokenID, nextPageToken := "", "", ""
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gDriveDirectoryMimeType)
l, err := s.list(nextPageToken, q)
if err != nil {
return "", err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
tokenID = fi.Id
break
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return "", err
}
}
if filename == "" {
return tokenID, nil
} else if tokenID == "" {
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
}
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gDriveDirectoryMimeType)
l, err = s.list(nextPageToken, q)
if err != nil {
return "", err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
fileID = fi.Id
break
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return "", err
}
}
if fileID == "" {
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
}
return fileID, nil
}
// Type returns the storage type
func (s *GDrive) Type() string {
return "gdrive"
}
// Head retrieves content length of a file from storage
func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
var fi *drive.File
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil {
return
}
contentLength = uint64(fi.Size)
return
}
// Get retrieves a file from storage
func (s *GDrive) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
var fi *drive.File
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
if err != nil {
return
}
if !s.hasChecksum(fi) {
err = fmt.Errorf("cannot find file %s/%s", token, filename)
return
}
contentLength = uint64(fi.Size)
fileGetCall := s.service.Files.Get(fileID)
if rng != nil {
header := fileGetCall.Header()
header.Set("Range", rng.Range())
}
var res *http.Response
res, err = fileGetCall.Context(ctx).Download()
if err != nil {
return
}
if rng != nil {
reader = res.Body
rng.AcceptLength(contentLength)
return
}
reader = res.Body
return
}
// Delete removes a file from storage
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
_ = s.service.Files.Delete(metadata).Do()
var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}
err = s.service.Files.Delete(fileID).Context(ctx).Do()
return
}
// Purge cleans up the storage
func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
nextPageToken := ""
expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gDriveDirectoryMimeType)
l, err := s.list(nextPageToken, q)
if err != nil {
return err
}
for 0 < len(l.Files) {
for _, fi := range l.Files {
err = s.service.Files.Delete(fi.Id).Context(ctx).Do()
if err != nil {
return
}
}
if l.NextPageToken == "" {
break
}
l, err = s.list(l.NextPageToken, q)
if err != nil {
return
}
}
return
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *GDrive) IsNotExist(err error) bool {
if err == nil {
return false
}
if e, ok := err.(*googleapi.Error); ok {
return e.Code == http.StatusNotFound
}
return false
}
// Put saves a file on storage
func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
dirID, err := s.findID("", token)
if err != nil {
return err
}
if dirID == "" {
dir := &drive.File{
Name: token,
Parents: []string{s.rootID},
MimeType: gDriveDirectoryMimeType,
}
di, err := s.service.Files.Create(dir).Fields("id").Do()
if err != nil {
return err
}
dirID = di.Id
}
// Instantiate empty drive file
dst := &drive.File{
Name: filename,
Parents: []string{dirID},
MimeType: contentType,
}
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
if err != nil {
return err
}
return nil
}
func (s *GDrive) IsRangeSupported() bool { return true }
// Retrieve a token, saves the token, then returns the generated client.
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile)
tok, err := gDriveTokenFromFile(tokenFile)
if err != nil {
tok = getGDriveTokenFromWeb(ctx, config, logger)
saveGDriveToken(tokenFile, tok, logger)
}
return config.Client(ctx, tok)
}
// Request a token from the web, then returns the retrieved token.
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
var authCode string
if _, err := fmt.Scan(&authCode); err != nil {
logger.Fatalf("Unable to read authorization code %v", err)
}
tok, err := config.Exchange(ctx, authCode)
if err != nil {
logger.Fatalf("Unable to retrieve token from web %v", err)
}
return tok
}
// Retrieves a token from a local file.
func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
defer CloseCheck(f)
if err != nil {
return nil, err
}
tok := &oauth2.Token{}
err = json.NewDecoder(f).Decode(tok)
return tok, err
}
// Saves a token to a file path.
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
logger.Printf("Saving credential file to: %s\n", path)
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f)
if err != nil {
logger.Fatalf("Unable to cache oauth token: %v", err)
}
err = json.NewEncoder(f).Encode(token)
if err != nil {
logger.Fatalf("Unable to encode oauth token: %v", err)
}
}

138
server/storage/local.go Normal file
View File

@@ -0,0 +1,138 @@
package storage
import (
"context"
"fmt"
"io"
"log"
"os"
"path/filepath"
"time"
)
// LocalStorage is a local storage
type LocalStorage struct {
Storage
basedir string
logger *log.Logger
}
// NewLocalStorage is the factory for LocalStorage
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
return &LocalStorage{basedir: basedir, logger: logger}, nil
}
// Type returns the storage type
func (s *LocalStorage) Type() string {
return "local"
}
// Head retrieves content length of a file from storage
func (s *LocalStorage) Head(_ context.Context, token string, filename string) (contentLength uint64, err error) {
path := filepath.Join(s.basedir, token, filename)
var fi os.FileInfo
if fi, err = os.Lstat(path); err != nil {
return
}
contentLength = uint64(fi.Size())
return
}
// Get retrieves a file from storage
func (s *LocalStorage) Get(_ context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
path := filepath.Join(s.basedir, token, filename)
var file *os.File
// content type , content length
if file, err = os.Open(path); err != nil {
return
}
reader = file
var fi os.FileInfo
if fi, err = os.Lstat(path); err != nil {
return
}
contentLength = uint64(fi.Size())
if rng != nil {
contentLength = rng.AcceptLength(contentLength)
if _, err = file.Seek(int64(rng.Start), 0); err != nil {
return
}
}
return
}
// Delete removes a file from storage
func (s *LocalStorage) Delete(_ context.Context, token string, filename string) (err error) {
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
_ = os.Remove(metadata)
path := filepath.Join(s.basedir, token, filename)
err = os.Remove(path)
return
}
// Purge cleans up the storage
func (s *LocalStorage) Purge(_ context.Context, days time.Duration) (err error) {
err = filepath.Walk(s.basedir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if info.ModTime().Before(time.Now().Add(-1 * days)) {
err = os.Remove(path)
return err
}
return nil
})
return
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *LocalStorage) IsNotExist(err error) bool {
if err == nil {
return false
}
return os.IsNotExist(err)
}
// Put saves a file on storage
func (s *LocalStorage) Put(_ context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
var f io.WriteCloser
var err error
path := filepath.Join(s.basedir, token)
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
return err
}
f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
defer CloseCheck(f)
if err != nil {
return err
}
if _, err = io.Copy(f, reader); err != nil {
return err
}
return nil
}
func (s *LocalStorage) IsRangeSupported() bool { return true }

188
server/storage/s3.go Normal file
View File

@@ -0,0 +1,188 @@
package storage
import (
"context"
"fmt"
"io"
"log"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
// S3Storage is a storage backed by AWS S3
type S3Storage struct {
Storage
bucket string
session *session.Session
s3 *s3.S3
logger *log.Logger
purgeDays time.Duration
noMultipart bool
}
// NewS3Storage is the factory for S3Storage
func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
return &S3Storage{
bucket: bucketName,
s3: s3.New(sess),
session: sess,
logger: logger,
noMultipart: disableMultipart,
purgeDays: time.Duration(purgeDays*24) * time.Hour,
}, nil
}
// Type returns the storage type
func (s *S3Storage) Type() string {
return "s3"
}
// Head retrieves content length of a file from storage
func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
key := fmt.Sprintf("%s/%s", token, filename)
headRequest := &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
// content type , content length
response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
if err != nil {
return
}
if response.ContentLength != nil {
contentLength = uint64(*response.ContentLength)
}
return
}
// Purge cleans up the storage
func (s *S3Storage) Purge(context.Context, time.Duration) (err error) {
// NOOP expiration is set at upload time
return nil
}
// IsNotExist indicates if a file doesn't exist on storage
func (s *S3Storage) IsNotExist(err error) bool {
if err == nil {
return false
}
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchKey:
return true
}
}
return false
}
// Get retrieves a file from storage
func (s *S3Storage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
key := fmt.Sprintf("%s/%s", token, filename)
getRequest := &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
if rng != nil {
getRequest.Range = aws.String(rng.Range())
}
response, err := s.s3.GetObjectWithContext(ctx, getRequest)
if err != nil {
return
}
if response.ContentLength != nil {
contentLength = uint64(*response.ContentLength)
}
if rng != nil && response.ContentRange != nil {
rng.SetContentRange(*response.ContentRange)
}
reader = response.Body
return
}
// Delete removes a file from storage
func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
deleteRequest := &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(metadata),
}
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
if err != nil {
return
}
key := fmt.Sprintf("%s/%s", token, filename)
deleteRequest = &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
}
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
return
}
// Put saves a file on storage
func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, _ uint64) (err error) {
key := fmt.Sprintf("%s/%s", token, filename)
s.logger.Printf("Uploading file %s to S3 Bucket", filename)
var concurrency int
if !s.noMultipart {
concurrency = 20
} else {
concurrency = 1
}
// Create an uploader with the session and custom options
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
u.Concurrency = concurrency // default is 5
u.LeavePartsOnError = false
})
var expire *time.Time
if s.purgeDays.Hours() > 0 {
expire = aws.Time(time.Now().Add(s.purgeDays))
}
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: reader,
Expires: expire,
ContentType: aws.String(contentType),
})
return
}
func (s *S3Storage) IsRangeSupported() bool { return true }
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
return session.Must(session.NewSession(&aws.Config{
Region: aws.String(region),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
S3ForcePathStyle: aws.Bool(forcePathStyle),
}))
}

163
server/storage/storj.go Normal file
View File

@@ -0,0 +1,163 @@
package storage
import (
"context"
"errors"
"io"
"log"
"time"
"storj.io/common/fpath"
"storj.io/common/storj"
"storj.io/uplink"
)
// StorjStorage is a storage backed by Storj
type StorjStorage struct {
Storage
project *uplink.Project
bucket *uplink.Bucket
purgeDays time.Duration
logger *log.Logger
}
// NewStorjStorage is the factory for StorjStorage
func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
var instance StorjStorage
var err error
pCtx := context.TODO()
ctx := fpath.WithTempData(pCtx, "", true)
uplConf := &uplink.Config{
UserAgent: "transfer-sh",
}
parsedAccess, err := uplink.ParseAccess(access)
if err != nil {
return nil, err
}
instance.project, err = uplConf.OpenProject(ctx, parsedAccess)
if err != nil {
return nil, err
}
instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
if err != nil {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = instance.project.Close()
return nil, err
}
instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
instance.logger = logger
return &instance, nil
}
// Type returns the storage type
func (s *StorjStorage) Type() string {
return "storj"
}
// Head retrieves content length of a file from storage
func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
key := storj.JoinPaths(token, filename)
obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
if err != nil {
return 0, err
}
contentLength = uint64(obj.System.ContentLength)
return
}
// Get retrieves a file from storage
func (s *StorjStorage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Getting file %s from Storj Bucket", filename)
options := uplink.DownloadOptions{}
if rng != nil {
options.Offset = int64(rng.Start)
if rng.Limit > 0 {
options.Length = int64(rng.Limit)
}
}
download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, &options)
if err != nil {
return nil, 0, err
}
contentLength = uint64(download.Info().System.ContentLength)
if rng != nil {
contentLength = rng.AcceptLength(contentLength)
}
reader = download
return
}
// Delete removes a file from storage
func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Deleting file %s from Storj Bucket", filename)
_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
return
}
// Purge cleans up the storage
func (s *StorjStorage) Purge(context.Context, time.Duration) (err error) {
// NOOP expiration is set at upload time
return nil
}
// Put saves a file on storage
func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
key := storj.JoinPaths(token, filename)
s.logger.Printf("Uploading file %s to Storj Bucket", filename)
var uploadOptions *uplink.UploadOptions
if s.purgeDays.Hours() > 0 {
uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
}
writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
if err != nil {
return err
}
n, err := io.Copy(writer, reader)
if err != nil || uint64(n) != contentLength {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = writer.Abort()
return err
}
err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
if err != nil {
//Ignoring the error to return the one that occurred first, but try to clean up.
_ = writer.Abort()
return err
}
err = writer.Commit()
return err
}
func (s *StorjStorage) IsRangeSupported() bool { return true }
// IsNotExist indicates if a file doesn't exist on storage
func (s *StorjStorage) IsNotExist(err error) bool {
return errors.Is(err, uplink.ErrObjectNotFound)
}

View File

@@ -30,25 +30,12 @@ import (
"fmt"
"math"
"net/http"
"net/mail"
"strconv"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/golang/gddo/httputil/header"
)
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
return session.Must(session.NewSession(&aws.Config{
Region: aws.String(region),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
S3ForcePathStyle: aws.Bool(forcePathStyle),
}))
}
func formatNumber(format string, s uint64) string {
return renderFloat(format, float64(s))
}
@@ -201,10 +188,6 @@ func renderFloat(format string, n float64) string {
return signStr + intStr + decimalStr + fracStr
}
func renderInteger(format string, n int) string {
return renderFloat(format, float64(n))
}
// Request.RemoteAddress contains port, which we want to remove i.e.:
// "[::1]:58292" => "[::1]"
func ipAddrFromRemoteAddr(s string) string {
@@ -215,45 +198,16 @@ func ipAddrFromRemoteAddr(s string) string {
return s[:idx]
}
func getIPAddress(r *http.Request) string {
hdr := r.Header
hdrRealIP := hdr.Get("X-Real-Ip")
hdrForwardedFor := hdr.Get("X-Forwarded-For")
if hdrRealIP == "" && hdrForwardedFor == "" {
return ipAddrFromRemoteAddr(r.RemoteAddr)
}
if hdrForwardedFor != "" {
// X-Forwarded-For is potentially a list of addresses separated with ","
parts := strings.Split(hdrForwardedFor, ",")
for i, p := range parts {
parts[i] = strings.TrimSpace(p)
}
// TODO: should return first non-local address
return parts[0]
}
return hdrRealIP
}
func encodeRFC2047(s string) string {
// use mail's rfc2047 to encode any string
addr := mail.Address{
Name: s,
Address: "",
}
return strings.Trim(addr.String(), " <>")
}
func acceptsHTML(hdr http.Header) bool {
actual := header.ParseAccept(hdr, "Accept")
for _, s := range actual {
if s.Value == "text/html" {
return (true)
return true
}
}
return (false)
return false
}
func formatSize(size int64) string {
@@ -279,9 +233,3 @@ func formatSize(size int64) string {
getSuffix := suffixes[int(math.Floor(base))]
return fmt.Sprintf("%s %s", strconv.FormatFloat(newVal, 'f', -1, 64), getSuffix)
}
func CloseCheck(f func() error) {
if err := f(); err != nil {
fmt.Println("Received close error:", err)
}
}

View File

@@ -30,7 +30,7 @@ import (
"github.com/gorilla/mux"
virustotal "github.com/dutchcoders/go-virustotal"
"github.com/dutchcoders/go-virustotal"
)
func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
@@ -45,14 +45,14 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
if err != nil {
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
reader := r.Body
result, err := vt.Scan(filename, reader)
if err != nil {
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
s.logger.Println(result)