mirror of
https://github.com/dutchcoders/transfer.sh.git
synced 2026-02-03 14:13:26 +00:00
Compare commits
29 Commits
clamav-pre
...
accept-ran
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2a29083960 | ||
|
|
158e5487ee | ||
|
|
806286ab35 | ||
|
|
d49aee59ba | ||
|
|
e08225e5f8 | ||
|
|
8597f1d9eb | ||
|
|
9e8ce19cd1 | ||
|
|
2bda0a1e55 | ||
|
|
d9369e8b39 | ||
|
|
193f944829 | ||
|
|
ebc4097959 | ||
|
|
ca798ff6f6 | ||
|
|
31520b1afd | ||
|
|
3588502c50 | ||
|
|
31ad4e01e1 | ||
|
|
343427d3b9 | ||
|
|
64c7759126 | ||
|
|
21812d3efc | ||
|
|
35e794220b | ||
|
|
f06aef1c3e | ||
|
|
eeff2c88be | ||
|
|
92324798d5 | ||
|
|
b30b296ac8 | ||
|
|
bb0891cd7d | ||
|
|
9c31ceb2c5 | ||
|
|
597554a59e | ||
|
|
368431fb6b | ||
|
|
e3bb49993c | ||
|
|
cff0a88bf3 |
11
.github/workflows/build-docker-images.yml
vendored
11
.github/workflows/build-docker-images.yml
vendored
@@ -34,9 +34,11 @@ jobs:
|
||||
fi
|
||||
|
||||
TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
|
||||
TAGS_NOROOT="--tag ${DOCKER_IMAGE}:${VERSION}-noroot"
|
||||
|
||||
if [ $VERSION = edge -o $VERSION = nightly ]; then
|
||||
TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
|
||||
TAGS_NOROOT="$TAGS_NOROOT --tag ${DOCKER_IMAGE}:latest-noroot"
|
||||
fi
|
||||
|
||||
echo ::set-output name=docker_image::${DOCKER_IMAGE}
|
||||
@@ -46,6 +48,12 @@ jobs:
|
||||
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
|
||||
--build-arg VCS_REF=${GITHUB_SHA::8} \
|
||||
${TAGS} .
|
||||
echo ::set-output name=buildx_args_noroot::--platform ${DOCKER_PLATFORMS} \
|
||||
--build-arg VERSION=${VERSION} \
|
||||
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
|
||||
--build-arg VCS_REF=${GITHUB_SHA::8} \
|
||||
--build-arg RUNAS=noroot \
|
||||
${TAGS_NOROOT} .
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
@@ -64,6 +72,7 @@ jobs:
|
||||
name: Docker Buildx (build)
|
||||
run: |
|
||||
docker buildx build --no-cache --pull --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }}
|
||||
docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args_noroot }}
|
||||
-
|
||||
name: Docker Login
|
||||
if: success() && github.event_name != 'pull_request'
|
||||
@@ -77,11 +86,13 @@ jobs:
|
||||
if: success() && github.event_name != 'pull_request'
|
||||
run: |
|
||||
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
|
||||
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args_noroot }}
|
||||
-
|
||||
name: Docker Check Manifest
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
run: |
|
||||
docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}
|
||||
docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}-noroot
|
||||
-
|
||||
name: Clear
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
|
||||
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -107,7 +107,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: ^1.17
|
||||
go-version: ^1.18
|
||||
|
||||
- name: Get project dependencies
|
||||
run: go mod download
|
||||
|
||||
5
.github/workflows/test.yml
vendored
5
.github/workflows/test.yml
vendored
@@ -13,11 +13,10 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
go_version:
|
||||
- 1.13.x
|
||||
- 1.14.x
|
||||
- 1.15.x
|
||||
- 1.16.x
|
||||
- 1.17.x
|
||||
- 1.18.X
|
||||
name: Test with ${{ matrix.go_version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -36,7 +35,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.17
|
||||
go-version: 1.18
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
|
||||
@@ -21,5 +21,5 @@ This code of conduct applies both within project spaces and in public spaces whe
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant] (https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html
|
||||
|
||||
|
||||
18
Dockerfile
18
Dockerfile
@@ -14,12 +14,28 @@ ENV GO111MODULE=on
|
||||
# build & install server
|
||||
RUN CGO_ENABLED=0 go build -tags netgo -ldflags "-X github.com/dutchcoders/transfer.sh/cmd.Version=$(git describe --tags) -a -s -w -extldflags '-static'" -o /go/bin/transfersh
|
||||
|
||||
ARG PUID=5000 \
|
||||
PGID=5000 \
|
||||
RUNAS
|
||||
|
||||
RUN mkdir -p /tmp/useradd /tmp/empty && \
|
||||
if [ ! -z "$RUNAS" ]; then \
|
||||
echo "${RUNAS}:x:${PUID}:${PGID}::/nonexistent:/sbin/nologin" >> /tmp/useradd/passwd && \
|
||||
echo "${RUNAS}:!:::::::" >> /tmp/useradd/shadow && \
|
||||
echo "${RUNAS}:x:${PGID}:" >> /tmp/useradd/group && \
|
||||
echo "${RUNAS}:!::" >> /tmp/useradd/groupshadow; else touch /tmp/useradd/unused; fi
|
||||
|
||||
FROM scratch AS final
|
||||
LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>"
|
||||
ARG RUNAS
|
||||
|
||||
COPY --from=build /go/bin/transfersh /go/bin/transfersh
|
||||
COPY --from=build /tmp/empty /tmp
|
||||
COPY --from=build /tmp/useradd/* /etc/
|
||||
COPY --from=build --chown=${RUNAS} /go/bin/transfersh /go/bin/transfersh
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
USER ${RUNAS}
|
||||
|
||||
ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"]
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
5
Makefile
Normal file
5
Makefile
Normal file
@@ -0,0 +1,5 @@
|
||||
.PHONY: lint
|
||||
|
||||
lint:
|
||||
golangci-lint run --out-format=github-actions --config .golangci.yml
|
||||
|
||||
193
README.md
193
README.md
@@ -12,7 +12,7 @@ The service at transfersh.com is of unknown origin and reported as cloud malware
|
||||
|
||||
### Upload:
|
||||
```bash
|
||||
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt
|
||||
$ curl -v --upload-file ./hello.txt https://transfer.sh/hello.txt
|
||||
```
|
||||
|
||||
### Encrypt & Upload:
|
||||
@@ -53,8 +53,9 @@ $ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "Max-Days: 1"
|
||||
|
||||
The URL used to request the deletion of a file and returned as a response header.
|
||||
```bash
|
||||
curl -sD - --upload-file ./hello https://transfer.sh/hello.txt | grep 'X-Url-Delete'
|
||||
X-Url-Delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
|
||||
curl -sD - --upload-file ./hello.txt https://transfer.sh/hello.txt | grep -i -E 'transfer\.sh|x-url-delete'
|
||||
x-url-delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
|
||||
https://transfer.sh/hello.txt/BAYh0/hello.txt
|
||||
```
|
||||
|
||||
## Examples
|
||||
@@ -111,6 +112,7 @@ lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated
|
||||
log | path to log file| | LOG |
|
||||
cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS |
|
||||
clamav-host | host for clamav feature | | CLAMAV_HOST |
|
||||
perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |
|
||||
rate-limit | request per minute | | RATE_LIMIT |
|
||||
max-upload-size | max upload size in kilobytes | | MAX_UPLOAD_SIZE |
|
||||
purge-days | number of days after the uploads are purged automatically | | PURGE_DAYS |
|
||||
@@ -139,12 +141,31 @@ $ go build -o transfersh main.go
|
||||
|
||||
## Docker
|
||||
|
||||
For easy deployment, we've created a Docker container.
|
||||
For easy deployment, we've created an official Docker container. There are two variants, differing only by which user runs the process.
|
||||
|
||||
The default one will run as `root`:
|
||||
|
||||
```bash
|
||||
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/
|
||||
```
|
||||
|
||||
The one tagged with the suffix `-noroot` will use `5000` as both UID and GID:
|
||||
```bash
|
||||
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest-noroot --provider local --basedir /tmp/
|
||||
```
|
||||
|
||||
### Building the Container
|
||||
You can also build the container yourself. This allows you to choose which UID/GID will be used, e.g. when using NFS mounts:
|
||||
```bash
|
||||
# Build arguments:
|
||||
# * RUNAS: If empty, the container will run as root.
|
||||
# Set this to anything to enable UID/GID selection.
|
||||
# * PUID: UID of the process. Needs RUNAS != "". Defaults to 5000.
|
||||
# * PGID: GID of the process. Needs RUNAS != "". Defaults to 5000.
|
||||
|
||||
docker build -t transfer.sh-noroot --build-arg RUNAS=doesntmatter --build-arg PUID=1337 --build-arg PGID=1338 .
|
||||
```
|
||||
|
||||
## S3 Usage
|
||||
|
||||
For the usage with a AWS S3 Bucket, you just need to specify the following options:
|
||||
@@ -203,6 +224,170 @@ You need to create an OAuth Client id from console.cloud.google.com, download th
|
||||
|
||||
```go run main.go --provider gdrive --basedir /tmp/ --gdrive-client-json-filepath /[credential_dir] --gdrive-local-config-path [directory_to_save_config] ```
|
||||
|
||||
## Shell functions
|
||||
|
||||
### Bash and zsh (multiple files uploaded as zip archive)
|
||||
##### Add this to .bashrc or .zshrc or its equivalent
|
||||
```bash
|
||||
transfer(){ if [ $# -eq 0 ];then echo "No arguments specified.\nUsage:\n transfer <file|directory>\n ... | transfer <file_name>">&2;return 1;fi;if tty -s;then file="$1";file_name=$(basename "$file");if [ ! -e "$file" ];then echo "$file: No such file or directory">&2;return 1;fi;if [ -d "$file" ];then file_name="$file_name.zip" ,;(cd "$file"&&zip -r -q - .)|curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null,;else cat "$file"|curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null;fi;else file_name=$1;curl --progress-bar --upload-file "-" "https://transfer.sh/$file_name"|tee /dev/null;fi;}
|
||||
```
|
||||
|
||||
#### Now you can use transfer function
|
||||
```
|
||||
$ transfer hello.txt
|
||||
```
|
||||
|
||||
|
||||
### Bash and zsh (with delete url, delete token output and prompt before uploading)
|
||||
##### Add this to .bashrc or .zshrc or its equivalent
|
||||
|
||||
<details><summary>Expand</summary><p>
|
||||
|
||||
```bash
|
||||
transfer()
|
||||
{
|
||||
local file
|
||||
declare -a file_array
|
||||
file_array=("${@}")
|
||||
|
||||
if [[ "${file_array[@]}" == "" || "${1}" == "--help" || "${1}" == "-h" ]]
|
||||
then
|
||||
echo "${0} - Upload arbitrary files to \"transfer.sh\"."
|
||||
echo ""
|
||||
echo "Usage: ${0} [options] [<file>]..."
|
||||
echo ""
|
||||
echo "OPTIONS:"
|
||||
echo " -h, --help"
|
||||
echo " show this message"
|
||||
echo ""
|
||||
echo "EXAMPLES:"
|
||||
echo " Upload a single file from the current working directory:"
|
||||
echo " ${0} \"image.img\""
|
||||
echo ""
|
||||
echo " Upload multiple files from the current working directory:"
|
||||
echo " ${0} \"image.img\" \"image2.img\""
|
||||
echo ""
|
||||
echo " Upload a file from a different directory:"
|
||||
echo " ${0} \"/tmp/some_file\""
|
||||
echo ""
|
||||
echo " Upload all files from the current working directory. Be aware of the webserver's rate limiting!:"
|
||||
echo " ${0} *"
|
||||
echo ""
|
||||
echo " Upload a single file from the current working directory and filter out the delete token and download link:"
|
||||
echo " ${0} \"image.img\" | awk --field-separator=\": \" '/Delete token:/ { print \$2 } /Download link:/ { print \$2 }'"
|
||||
echo ""
|
||||
echo " Show help text from \"transfer.sh\":"
|
||||
echo " curl --request GET \"https://transfer.sh\""
|
||||
return 0
|
||||
else
|
||||
for file in "${file_array[@]}"
|
||||
do
|
||||
if [[ ! -f "${file}" ]]
|
||||
then
|
||||
echo -e "\e[01;31m'${file}' could not be found or is not a file.\e[0m" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
unset file
|
||||
fi
|
||||
|
||||
local upload_files
|
||||
local curl_output
|
||||
local awk_output
|
||||
|
||||
du -c -k -L "${file_array[@]}" >&2
|
||||
# be compatible with "bash"
|
||||
if [[ "${ZSH_NAME}" == "zsh" ]]
|
||||
then
|
||||
read $'upload_files?\e[01;31mDo you really want to upload the above files ('"${#file_array[@]}"$') to "transfer.sh"? (Y/n): \e[0m'
|
||||
elif [[ "${BASH}" == *"bash"* ]]
|
||||
then
|
||||
read -p $'\e[01;31mDo you really want to upload the above files ('"${#file_array[@]}"$') to "transfer.sh"? (Y/n): \e[0m' upload_files
|
||||
fi
|
||||
|
||||
case "${upload_files:-y}" in
|
||||
"y"|"Y")
|
||||
# for the sake of the progress bar, execute "curl" for each file.
|
||||
# the parameters "--include" and "--form" will suppress the progress bar.
|
||||
for file in "${file_array[@]}"
|
||||
do
|
||||
# show delete link and filter out the delete token from the response header after upload.
|
||||
# it is important to save "curl's" "stdout" via a subshell to a variable or redirect it to another command,
|
||||
# which just redirects to "stdout" in order to have a sane output afterwards.
|
||||
# the progress bar is redirected to "stderr" and is only displayed,
|
||||
# if "stdout" is redirected to something; e.g. ">/dev/null", "tee /dev/null" or "| <some_command>".
|
||||
# the response header is redirected to "stdout", so redirecting "stdout" to "/dev/null" does not make any sense.
|
||||
# redirecting "curl's" "stderr" to "stdout" ("2>&1") will suppress the progress bar.
|
||||
curl_output=$(curl --request PUT --progress-bar --dump-header - --upload-file "${file}" "https://transfer.sh/")
|
||||
awk_output=$(awk \
|
||||
'gsub("\r", "", $0) && tolower($1) ~ /x-url-delete/ \
|
||||
{
|
||||
delete_link=$2;
|
||||
print "Delete command: curl --request DELETE " "\""delete_link"\"";
|
||||
|
||||
gsub(".*/", "", delete_link);
|
||||
delete_token=delete_link;
|
||||
print "Delete token: " delete_token;
|
||||
}
|
||||
|
||||
END{
|
||||
print "Download link: " $0;
|
||||
}' <<< "${curl_output}")
|
||||
|
||||
# return the results via "stdout", "awk" does not do this for some reason.
|
||||
echo -e "${awk_output}\n"
|
||||
|
||||
# avoid rate limiting as much as possible; nginx: too many requests.
|
||||
if (( ${#file_array[@]} > 4 ))
|
||||
then
|
||||
sleep 5
|
||||
fi
|
||||
done
|
||||
;;
|
||||
|
||||
"n"|"N")
|
||||
return 1
|
||||
;;
|
||||
|
||||
*)
|
||||
echo -e "\e[01;31mWrong input: '${upload_files}'.\e[0m" >&2
|
||||
return 1
|
||||
esac
|
||||
}
|
||||
```
|
||||
|
||||
</p></details>
|
||||
|
||||
#### Sample output
|
||||
```bash
|
||||
$ ls -lh
|
||||
total 20M
|
||||
-rw-r--r-- 1 <some_username> <some_username> 10M Apr 4 21:08 image.img
|
||||
-rw-r--r-- 1 <some_username> <some_username> 10M Apr 4 21:08 image2.img
|
||||
$ transfer image*
|
||||
10240K image2.img
|
||||
10240K image.img
|
||||
20480K total
|
||||
Do you really want to upload the above files (2) to "transfer.sh"? (Y/n):
|
||||
######################################################################################################################################################################################################################################## 100.0%
|
||||
Delete command: curl --request DELETE "https://transfer.sh/wJw9pz/image2.img/mSctGx7pYCId"
|
||||
Delete token: mSctGx7pYCId
|
||||
Download link: https://transfer.sh/wJw9pz/image2.img
|
||||
|
||||
######################################################################################################################################################################################################################################## 100.0%
|
||||
Delete command: curl --request DELETE "https://transfer.sh/ljJc5I/image.img/nw7qaoiKUwCU"
|
||||
Delete token: nw7qaoiKUwCU
|
||||
Download link: https://transfer.sh/ljJc5I/image.img
|
||||
|
||||
$ transfer "image.img" | awk --field-separator=": " '/Delete token:/ { print $2 } /Download link:/ { print $2 }'
|
||||
10240K image.img
|
||||
10240K total
|
||||
Do you really want to upload the above files (1) to "transfer.sh"? (Y/n):
|
||||
######################################################################################################################################################################################################################################## 100.0%
|
||||
tauN5dE3fWJe
|
||||
https://transfer.sh/MYkuqn/image.img
|
||||
```
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are welcome.
|
||||
|
||||
44
cmd/cmd.go
44
cmd/cmd.go
@@ -2,6 +2,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/dutchcoders/transfer.sh/server/storage"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -240,6 +241,11 @@ var globalFlags = []cli.Flag{
|
||||
Value: "",
|
||||
EnvVar: "CLAMAV_HOST",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "perform-clamav-prescan",
|
||||
Usage: "perform-clamav-prescan",
|
||||
EnvVar: "PERFORM_CLAMAV_PRESCAN",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "virustotal-key",
|
||||
Usage: "virustotal-key",
|
||||
@@ -294,8 +300,8 @@ type Cmd struct {
|
||||
*cli.App
|
||||
}
|
||||
|
||||
func versionAction(c *cli.Context) {
|
||||
fmt.Println(color.YellowString(fmt.Sprintf("transfer.sh %s: Easy file sharing from the command line", Version)))
|
||||
func versionCommand(_ *cli.Context) {
|
||||
fmt.Println(color.YellowString("transfer.sh %s: Easy file sharing from the command line", Version))
|
||||
}
|
||||
|
||||
// New is the factory for transfer.sh
|
||||
@@ -313,7 +319,7 @@ func New() *Cmd {
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "version",
|
||||
Action: versionAction,
|
||||
Action: versionCommand,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -322,7 +328,7 @@ func New() *Cmd {
|
||||
}
|
||||
|
||||
app.Action = func(c *cli.Context) {
|
||||
options := []server.OptionFn{}
|
||||
var options []server.OptionFn
|
||||
if v := c.String("listener"); v != "" {
|
||||
options = append(options, server.Listener(v))
|
||||
}
|
||||
@@ -388,6 +394,14 @@ func New() *Cmd {
|
||||
options = append(options, server.ClamavHost(v))
|
||||
}
|
||||
|
||||
if v := c.Bool("perform-clamav-prescan"); v {
|
||||
if c.String("clamav-host") == "" {
|
||||
panic("clamav-host not set")
|
||||
}
|
||||
|
||||
options = append(options, server.PerformClamavPrescan(v))
|
||||
}
|
||||
|
||||
if v := c.Int64("max-upload-size"); v > 0 {
|
||||
options = append(options, server.MaxUploadSize(v))
|
||||
}
|
||||
@@ -450,42 +464,42 @@ func New() *Cmd {
|
||||
panic("secret-key not set.")
|
||||
} else if bucket := c.String("bucket"); bucket == "" {
|
||||
panic("bucket not set.")
|
||||
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
|
||||
} else if store, err := storage.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
|
||||
panic(err)
|
||||
} else {
|
||||
options = append(options, server.UseStorage(storage))
|
||||
options = append(options, server.UseStorage(store))
|
||||
}
|
||||
case "gdrive":
|
||||
chunkSize := c.Int("gdrive-chunk-size")
|
||||
chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024
|
||||
|
||||
if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
|
||||
panic("client-json-filepath not set.")
|
||||
panic("gdrive-client-json-filepath not set.")
|
||||
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
|
||||
panic("local-config-path not set.")
|
||||
panic("gdrive-local-config-path not set.")
|
||||
} else if basedir := c.String("basedir"); basedir == "" {
|
||||
panic("basedir not set.")
|
||||
} else if storage, err := server.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
|
||||
} else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
|
||||
panic(err)
|
||||
} else {
|
||||
options = append(options, server.UseStorage(storage))
|
||||
options = append(options, server.UseStorage(store))
|
||||
}
|
||||
case "storj":
|
||||
if access := c.String("storj-access"); access == "" {
|
||||
panic("storj-access not set.")
|
||||
} else if bucket := c.String("storj-bucket"); bucket == "" {
|
||||
panic("storj-bucket not set.")
|
||||
} else if storage, err := server.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
|
||||
} else if store, err := storage.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
|
||||
panic(err)
|
||||
} else {
|
||||
options = append(options, server.UseStorage(storage))
|
||||
options = append(options, server.UseStorage(store))
|
||||
}
|
||||
case "local":
|
||||
if v := c.String("basedir"); v == "" {
|
||||
panic("basedir not set.")
|
||||
} else if storage, err := server.NewLocalStorage(v, logger); err != nil {
|
||||
} else if store, err := storage.NewLocalStorage(v, logger); err != nil {
|
||||
panic(err)
|
||||
} else {
|
||||
options = append(options, server.UseStorage(storage))
|
||||
options = append(options, server.UseStorage(store))
|
||||
}
|
||||
default:
|
||||
panic("Provider not set or invalid.")
|
||||
|
||||
26
go.mod
26
go.mod
@@ -1,20 +1,23 @@
|
||||
module github.com/dutchcoders/transfer.sh
|
||||
|
||||
go 1.13
|
||||
go 1.15
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.77.0 // indirect
|
||||
cloud.google.com/go/compute v1.18.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14
|
||||
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2
|
||||
github.com/aws/aws-sdk-go v1.37.14
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
|
||||
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329
|
||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3
|
||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20220824020025-7240e75c3bb8
|
||||
github.com/elazarl/go-bindata-assetfs v1.0.1
|
||||
github.com/fatih/color v1.10.0
|
||||
github.com/garyburd/redigo v1.6.2 // indirect
|
||||
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.2 // indirect
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/securecookie v1.1.1 // indirect
|
||||
@@ -24,13 +27,14 @@ require (
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
|
||||
github.com/urfave/cli v1.22.5
|
||||
go.opencensus.io v0.22.6 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
|
||||
google.golang.org/api v0.40.0
|
||||
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
|
||||
golang.org/x/net v0.6.0 // indirect
|
||||
golang.org/x/oauth2 v0.5.0
|
||||
google.golang.org/api v0.109.0
|
||||
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
|
||||
google.golang.org/grpc v1.53.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
|
||||
storj.io/common v0.0.0-20211102144601-401a79f0706a
|
||||
storj.io/uplink v1.7.1
|
||||
storj.io/common v0.0.0-20220405183405-ffdc3ab808c6
|
||||
storj.io/uplink v1.8.2
|
||||
)
|
||||
|
||||
@@ -27,18 +27,19 @@ THE SOFTWARE.
|
||||
package server
|
||||
|
||||
import (
|
||||
// _ "transfer.sh/app/handlers"
|
||||
// _ "transfer.sh/app/utils"
|
||||
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
clamd "github.com/dutchcoders/go-clamd"
|
||||
|
||||
"github.com/dutchcoders/go-clamd"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
const clamavScanStatusOK = "OK"
|
||||
|
||||
func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
|
||||
@@ -49,23 +50,53 @@ func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
s.logger.Printf("Scanning %s %d %s", filename, contentLength, contentType)
|
||||
|
||||
reader := r.Body
|
||||
|
||||
c := clamd.NewClamd(s.ClamAVDaemonHost)
|
||||
|
||||
abort := make(chan bool)
|
||||
defer close(abort)
|
||||
response, err := c.ScanStream(reader, abort)
|
||||
file, err := ioutil.TempFile(s.tempPath, "clamav-")
|
||||
defer s.cleanTmpFile(file)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.Copy(file, r.Body)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
status, err := s.performScan(file.Name())
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", status)))
|
||||
}
|
||||
|
||||
func (s *Server) performScan(path string) (string, error) {
|
||||
c := clamd.NewClamd(s.ClamAVDaemonHost)
|
||||
|
||||
responseCh := make(chan chan *clamd.ScanResult)
|
||||
errCh := make(chan error)
|
||||
go func(responseCh chan chan *clamd.ScanResult, errCh chan error) {
|
||||
response, err := c.ScanFile(path)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
responseCh <- response
|
||||
}(responseCh, errCh)
|
||||
|
||||
select {
|
||||
case s := <-response:
|
||||
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", s.Status)))
|
||||
case err := <-errCh:
|
||||
return "", err
|
||||
case response := <-responseCh:
|
||||
st := <-response
|
||||
return st.Status, nil
|
||||
case <-time.After(time.Second * 60):
|
||||
abort <- true
|
||||
return "", errors.New("clamav scan timeout")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
html_template "html/template"
|
||||
htmlTemplate "html/template"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
@@ -50,14 +50,17 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
text_template "text/template"
|
||||
textTemplate "text/template"
|
||||
"time"
|
||||
|
||||
"github.com/dutchcoders/transfer.sh/server/storage"
|
||||
|
||||
web "github.com/dutchcoders/transfer.sh-web"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
blackfriday "github.com/russross/blackfriday/v2"
|
||||
"github.com/russross/blackfriday/v2"
|
||||
"github.com/skip2/go-qrcode"
|
||||
"golang.org/x/net/idna"
|
||||
)
|
||||
|
||||
const getPathPart = "get"
|
||||
@@ -71,24 +74,24 @@ func stripPrefix(path string) string {
|
||||
return strings.Replace(path, web.Prefix+"/", "", -1)
|
||||
}
|
||||
|
||||
func initTextTemplates() *text_template.Template {
|
||||
templateMap := text_template.FuncMap{"format": formatNumber}
|
||||
func initTextTemplates() *textTemplate.Template {
|
||||
templateMap := textTemplate.FuncMap{"format": formatNumber}
|
||||
|
||||
// Templates with functions available to them
|
||||
var templates = text_template.New("").Funcs(templateMap)
|
||||
var templates = textTemplate.New("").Funcs(templateMap)
|
||||
return templates
|
||||
}
|
||||
|
||||
func initHTMLTemplates() *html_template.Template {
|
||||
templateMap := html_template.FuncMap{"format": formatNumber}
|
||||
func initHTMLTemplates() *htmlTemplate.Template {
|
||||
templateMap := htmlTemplate.FuncMap{"format": formatNumber}
|
||||
|
||||
// Templates with functions available to them
|
||||
var templates = html_template.New("").Funcs(templateMap)
|
||||
var templates = htmlTemplate.New("").Funcs(templateMap)
|
||||
|
||||
return templates
|
||||
}
|
||||
|
||||
func healthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func healthHandler(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("Approaching Neutral Zone, all systems normal and functioning."))
|
||||
}
|
||||
|
||||
@@ -136,7 +139,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
var templatePath string
|
||||
var content html_template.HTML
|
||||
var content htmlTemplate.HTML
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(contentType, "image/"):
|
||||
@@ -149,7 +152,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
templatePath = "download.markdown.html"
|
||||
|
||||
var reader io.ReadCloser
|
||||
if reader, _, err = s.storage.Get(r.Context(), token, filename); err != nil {
|
||||
if reader, _, err = s.storage.Get(r.Context(), token, filename, nil); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
@@ -164,9 +167,9 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(contentType, "text/x-markdown") || strings.HasPrefix(contentType, "text/markdown") {
|
||||
unsafe := blackfriday.Run(data)
|
||||
output := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
|
||||
content = html_template.HTML(output)
|
||||
content = htmlTemplate.HTML(output)
|
||||
} else if strings.HasPrefix(contentType, "text/plain") {
|
||||
content = html_template.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
|
||||
content = htmlTemplate.HTML(fmt.Sprintf("<pre>%s</pre>", html.EscapeString(string(data))))
|
||||
} else {
|
||||
templatePath = "download.sandbox.html"
|
||||
}
|
||||
@@ -193,7 +196,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
data := struct {
|
||||
ContentType string
|
||||
Content html_template.HTML
|
||||
Content htmlTemplate.HTML
|
||||
Filename string
|
||||
URL string
|
||||
URLGet string
|
||||
@@ -280,7 +283,7 @@ func (s *Server) viewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *Server) notFoundHandler(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, http.StatusText(404), 404)
|
||||
}
|
||||
|
||||
@@ -291,7 +294,7 @@ func sanitize(fileName string) string {
|
||||
func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(_24K); nil != err {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Error occurred copying to output stream", 500)
|
||||
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -299,93 +302,103 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
|
||||
for _, fheaders := range r.MultipartForm.File {
|
||||
for _, fheader := range fheaders {
|
||||
filename := sanitize(fheader.Filename)
|
||||
contentType := mime.TypeByExtension(filepath.Ext(fheader.Filename))
|
||||
responseBody := ""
|
||||
|
||||
for _, fHeaders := range r.MultipartForm.File {
|
||||
for _, fHeader := range fHeaders {
|
||||
filename := sanitize(fHeader.Filename)
|
||||
contentType := mime.TypeByExtension(filepath.Ext(fHeader.Filename))
|
||||
|
||||
var f io.Reader
|
||||
var err error
|
||||
|
||||
if f, err = fheader.Open(); err != nil {
|
||||
if f, err = fHeader.Open(); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
file, err := ioutil.TempFile(s.tempPath, "transfer-")
|
||||
defer s.cleanTmpFile(file)
|
||||
|
||||
n, err := io.CopyN(&b, f, _24K+1)
|
||||
if err != nil && err != io.EOF {
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var file *os.File
|
||||
var reader io.Reader
|
||||
|
||||
if n > _24K {
|
||||
file, err = ioutil.TempFile(s.tempPath, "transfer-")
|
||||
defer s.cleanTmpFile(file)
|
||||
if err != nil {
|
||||
s.logger.Fatal(err)
|
||||
}
|
||||
|
||||
n, err = io.Copy(file, io.MultiReader(&b, f))
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
reader, err = os.Open(file.Name())
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
reader = bytes.NewReader(b.Bytes())
|
||||
n, err := io.Copy(file, f)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
contentLength := n
|
||||
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if s.maxUploadSize > 0 && contentLength > s.maxUploadSize {
|
||||
s.logger.Print("Entity too large")
|
||||
http.Error(w, http.StatusText(http.StatusRequestEntityTooLarge), http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
|
||||
if s.performClamavPrescan {
|
||||
status, err := s.performScan(file.Name())
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if status != clamavScanStatusOK {
|
||||
s.logger.Printf("prescan positive: %s", status)
|
||||
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
metadata := metadataForRequest(contentType, s.randomTokenLength, r)
|
||||
|
||||
buffer := &bytes.Buffer{}
|
||||
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not encode metadata", 500)
|
||||
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not save metadata", 500)
|
||||
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
|
||||
|
||||
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
|
||||
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
|
||||
s.logger.Printf("Backend storage error: %s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
filename = url.PathEscape(filename)
|
||||
relativeURL, _ := url.Parse(path.Join(s.proxyPath, token, filename))
|
||||
_, _ = w.Write([]byte(getURL(r, s.proxyPort).ResolveReference(relativeURL).String()))
|
||||
deleteURL, _ := url.Parse(path.Join(s.proxyPath, token, filename, metadata.DeletionToken))
|
||||
w.Header().Add("X-Url-Delete", resolveURL(r, deleteURL, s.proxyPort))
|
||||
responseBody += fmt.Sprintln(getURL(r, s.proxyPort).ResolveReference(relativeURL).String())
|
||||
}
|
||||
}
|
||||
_, err := w.Write([]byte(responseBody))
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) cleanTmpFile(f *os.File) {
|
||||
@@ -448,56 +461,35 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
contentLength := r.ContentLength
|
||||
|
||||
var reader io.Reader
|
||||
defer storage.CloseCheck(r.Body)
|
||||
|
||||
reader = r.Body
|
||||
file, err := ioutil.TempFile(s.tempPath, "transfer-")
|
||||
defer s.cleanTmpFile(file)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
defer CloseCheck(r.Body.Close)
|
||||
// queue file to disk, because s3 needs content length
|
||||
// and clamav prescan scans a file
|
||||
n, err := io.Copy(file, r.Body)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
|
||||
if contentLength == -1 {
|
||||
// queue file to disk, because s3 needs content length
|
||||
var err error
|
||||
return
|
||||
}
|
||||
|
||||
f := reader
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Cannot reset cache file", http.StatusInternalServerError)
|
||||
|
||||
var b bytes.Buffer
|
||||
|
||||
n, err := io.CopyN(&b, f, _24K+1)
|
||||
if err != nil && err != io.EOF {
|
||||
s.logger.Printf("Error putting new file: %s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
var file *os.File
|
||||
|
||||
if n > _24K {
|
||||
file, err = ioutil.TempFile(s.tempPath, "transfer-")
|
||||
defer s.cleanTmpFile(file)
|
||||
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
n, err = io.Copy(file, io.MultiReader(&b, f))
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
reader, err = os.Open(file.Name())
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
reader = bytes.NewReader(b.Bytes())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if contentLength < 1 {
|
||||
contentLength = n
|
||||
}
|
||||
|
||||
@@ -509,10 +501,25 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if contentLength == 0 {
|
||||
s.logger.Print("Empty content-length")
|
||||
http.Error(w, "Could not upload empty file", 400)
|
||||
http.Error(w, "Could not upload empty file", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if s.performClamavPrescan {
|
||||
status, err := s.performScan(file.Name())
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if status != clamavScanStatusOK {
|
||||
s.logger.Printf("prescan positive: %s", status)
|
||||
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(vars["filename"]))
|
||||
|
||||
token := token(s.randomTokenLength)
|
||||
@@ -522,25 +529,23 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
|
||||
buffer := &bytes.Buffer{}
|
||||
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not encode metadata", 500)
|
||||
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
|
||||
return
|
||||
} else if !metadata.MaxDate.IsZero() && time.Now().After(metadata.MaxDate) {
|
||||
s.logger.Print("Invalid MaxDate")
|
||||
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", 400)
|
||||
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", http.StatusBadRequest)
|
||||
return
|
||||
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not save metadata", 500)
|
||||
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
|
||||
|
||||
var err error
|
||||
|
||||
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
|
||||
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
|
||||
s.logger.Printf("Error putting new file: %s", err.Error())
|
||||
http.Error(w, "Could not save file", 500)
|
||||
http.Error(w, "Could not save file", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -622,6 +627,14 @@ func getURL(r *http.Request, proxyPort string) *url.URL {
|
||||
host = r.Host
|
||||
port = ""
|
||||
}
|
||||
|
||||
p := idna.New(idna.ValidateForRegistration())
|
||||
var hostFromPunycode string
|
||||
hostFromPunycode, err = p.ToUnicode(host)
|
||||
if err == nil {
|
||||
host = hostFromPunycode
|
||||
}
|
||||
|
||||
if len(proxyPort) != 0 {
|
||||
port = proxyPort
|
||||
}
|
||||
@@ -680,8 +693,8 @@ func (s *Server) checkMetadata(ctx context.Context, token, filename string, incr
|
||||
|
||||
var metadata metadata
|
||||
|
||||
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
|
||||
defer CloseCheck(r.Close)
|
||||
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
|
||||
defer storage.CloseCheck(r)
|
||||
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
@@ -716,8 +729,8 @@ func (s *Server) checkDeletionToken(ctx context.Context, deletionToken, token, f
|
||||
|
||||
var metadata metadata
|
||||
|
||||
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename))
|
||||
defer CloseCheck(r.Close)
|
||||
r, _, err := s.storage.Get(ctx, token, fmt.Sprintf("%s.metadata", filename), nil)
|
||||
defer storage.CloseCheck(r)
|
||||
|
||||
if s.storage.IsNotExist(err) {
|
||||
return errors.New("metadata doesn't exist")
|
||||
@@ -766,7 +779,7 @@ func (s *Server) deleteHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
} else if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not delete file.", 500)
|
||||
http.Error(w, "Could not delete file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -779,8 +792,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
|
||||
zipfilename := fmt.Sprintf("transfersh-%d.zip", uint16(time.Now().UnixNano()))
|
||||
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", zipfilename))
|
||||
w.Header().Set("Connection", "close")
|
||||
commonHeader(w, zipfilename)
|
||||
|
||||
zw := zip.NewWriter(w)
|
||||
|
||||
@@ -795,8 +807,8 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
|
||||
reader, _, err := s.storage.Get(r.Context(), token, filename)
|
||||
defer CloseCheck(reader.Close)
|
||||
reader, _, err := s.storage.Get(r.Context(), token, filename, nil)
|
||||
defer storage.CloseCheck(reader)
|
||||
|
||||
if err != nil {
|
||||
if s.storage.IsNotExist(err) {
|
||||
@@ -805,7 +817,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not retrieve file.", 500)
|
||||
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -820,20 +832,20 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = io.Copy(fw, reader); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := zw.Close(); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -846,14 +858,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
|
||||
tarfilename := fmt.Sprintf("transfersh-%d.tar.gz", uint16(time.Now().UnixNano()))
|
||||
|
||||
w.Header().Set("Content-Type", "application/x-gzip")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", tarfilename))
|
||||
w.Header().Set("Connection", "close")
|
||||
commonHeader(w, tarfilename)
|
||||
|
||||
gw := gzip.NewWriter(w)
|
||||
defer CloseCheck(gw.Close)
|
||||
defer storage.CloseCheck(gw)
|
||||
|
||||
zw := tar.NewWriter(gw)
|
||||
defer CloseCheck(zw.Close)
|
||||
defer storage.CloseCheck(zw)
|
||||
|
||||
for _, key := range strings.Split(files, ",") {
|
||||
key = resolveKey(key, s.proxyPath)
|
||||
@@ -866,8 +877,8 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
|
||||
defer CloseCheck(reader.Close)
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
|
||||
defer storage.CloseCheck(reader)
|
||||
|
||||
if err != nil {
|
||||
if s.storage.IsNotExist(err) {
|
||||
@@ -876,7 +887,7 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not retrieve file.", 500)
|
||||
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -888,13 +899,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
|
||||
err = zw.WriteHeader(header)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = io.Copy(zw, reader); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -908,11 +919,10 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
|
||||
tarfilename := fmt.Sprintf("transfersh-%d.tar", uint16(time.Now().UnixNano()))
|
||||
|
||||
w.Header().Set("Content-Type", "application/x-tar")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", tarfilename))
|
||||
w.Header().Set("Connection", "close")
|
||||
commonHeader(w, tarfilename)
|
||||
|
||||
zw := tar.NewWriter(w)
|
||||
defer CloseCheck(zw.Close)
|
||||
defer storage.CloseCheck(zw)
|
||||
|
||||
for _, key := range strings.Split(files, ",") {
|
||||
key = resolveKey(key, s.proxyPath)
|
||||
@@ -925,8 +935,8 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
|
||||
defer CloseCheck(reader.Close)
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, nil)
|
||||
defer storage.CloseCheck(reader)
|
||||
|
||||
if err != nil {
|
||||
if s.storage.IsNotExist(err) {
|
||||
@@ -935,7 +945,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not retrieve file.", 500)
|
||||
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -947,13 +957,13 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
|
||||
err = zw.WriteHeader(header)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = io.Copy(zw, reader); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Internal server error.", 500)
|
||||
http.Error(w, "Internal server error.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -980,7 +990,7 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
} else if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not retrieve file.", 500)
|
||||
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -991,6 +1001,10 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Connection", "close")
|
||||
w.Header().Set("X-Remaining-Downloads", remainingDownloads)
|
||||
w.Header().Set("X-Remaining-Days", remainingDays)
|
||||
|
||||
if s.storage.IsRangeSupported() {
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1008,23 +1022,49 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var rng *storage.Range
|
||||
if r.Header.Get("Range") != "" {
|
||||
rng = storage.ParseRange(r.Header.Get("Range"))
|
||||
}
|
||||
|
||||
contentType := metadata.ContentType
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename)
|
||||
defer CloseCheck(reader.Close)
|
||||
reader, contentLength, err := s.storage.Get(r.Context(), token, filename, rng)
|
||||
defer storage.CloseCheck(reader)
|
||||
|
||||
rdr := io.Reader(reader)
|
||||
|
||||
if s.storage.IsNotExist(err) {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
return
|
||||
} else if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Could not retrieve file.", 500)
|
||||
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if rng != nil {
|
||||
cr := rng.ContentRange()
|
||||
if cr != "" {
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("Content-Range", cr)
|
||||
if rng.Limit > 0 {
|
||||
rdr = io.LimitReader(reader, int64(rng.Limit))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var disposition string
|
||||
|
||||
if action == "inline" {
|
||||
disposition = "inline"
|
||||
/*
|
||||
metadata.ContentType is unable to determine the type of the content,
|
||||
metadata.ContentType is unable to determine the type of the content,
|
||||
metadata.ContentType is unable to determine the type of the content,
|
||||
So add text/plain in this case to fix XSS related issues/
|
||||
*/
|
||||
if strings.TrimSpace(contentType) == "" {
|
||||
contentType = "text/plain"
|
||||
}
|
||||
} else {
|
||||
disposition = "attachment"
|
||||
}
|
||||
@@ -1035,41 +1075,31 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Length", strconv.FormatUint(contentLength, 10))
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", disposition, filename))
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("X-Remaining-Downloads", remainingDownloads)
|
||||
w.Header().Set("X-Remaining-Days", remainingDays)
|
||||
|
||||
if rng != nil && rng.ContentRange() != "" {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
}
|
||||
|
||||
if disposition == "inline" && canContainsXSS(contentType) {
|
||||
reader = ioutil.NopCloser(bluemonday.UGCPolicy().SanitizeReader(reader))
|
||||
}
|
||||
|
||||
if w.Header().Get("Range") != "" || strings.HasPrefix(metadata.ContentType, "video") || strings.HasPrefix(metadata.ContentType, "audio") {
|
||||
file, err := ioutil.TempFile(s.tempPath, "range-")
|
||||
defer s.cleanTmpFile(file)
|
||||
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Error occurred copying to output stream", 500)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.Copy(file, reader)
|
||||
if err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Error occurred copying to output stream", 500)
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeContent(w, r, filename, time.Now(), file)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = io.Copy(w, reader); err != nil {
|
||||
if _, err = io.Copy(w, rdr); err != nil {
|
||||
s.logger.Printf("%s", err.Error())
|
||||
http.Error(w, "Error occurred copying to output stream", 500)
|
||||
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func commonHeader(w http.ResponseWriter, filename string) {
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
|
||||
w.Header().Set("Connection", "close")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
}
|
||||
|
||||
// RedirectHandler handles redirect
|
||||
func (s *Server) RedirectHandler(h http.Handler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1136,12 +1166,12 @@ func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
|
||||
|
||||
username, password, authOK := r.BasicAuth()
|
||||
if !authOK {
|
||||
http.Error(w, "Not authorized", 401)
|
||||
http.Error(w, "Not authorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
if username != s.AuthUser || password != s.AuthPass {
|
||||
http.Error(w, "Not authorized", 401)
|
||||
http.Error(w, "Not authorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -88,10 +88,10 @@ func (f *ipFilter) BlockIP(ip string) bool {
|
||||
}
|
||||
|
||||
func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
|
||||
//check if has subnet
|
||||
if ip, net, err := net.ParseCIDR(str); err == nil {
|
||||
//check if provided string describes a subnet
|
||||
if ip, network, err := net.ParseCIDR(str); err == nil {
|
||||
// containing only one ip?
|
||||
if n, total := net.Mask.Size(); n == total {
|
||||
if n, total := network.Mask.Size(); n == total {
|
||||
f.mut.Lock()
|
||||
f.ips[ip.String()] = allowed
|
||||
f.mut.Unlock()
|
||||
@@ -110,7 +110,7 @@ func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
|
||||
if !found {
|
||||
f.subnets = append(f.subnets, &subnet{
|
||||
str: str,
|
||||
ipnet: net,
|
||||
ipnet: network,
|
||||
allowed: allowed,
|
||||
})
|
||||
}
|
||||
@@ -182,7 +182,7 @@ func (f *ipFilter) NetBlocked(ip net.IP) bool {
|
||||
return !f.NetAllowed(ip)
|
||||
}
|
||||
|
||||
//WrapIPFilter the provided handler with simple IP blocking middleware
|
||||
//Wrap the provided handler with simple IP blocking middleware
|
||||
//using this IP filter and its configuration
|
||||
func (f *ipFilter) Wrap(next http.Handler) http.Handler {
|
||||
return &ipFilterMiddleware{ipFilter: f, next: next}
|
||||
|
||||
@@ -25,39 +25,35 @@ THE SOFTWARE.
|
||||
package server
|
||||
|
||||
import (
|
||||
crypto_rand "crypto/rand"
|
||||
"context"
|
||||
cryptoRand "crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
gorillaHandlers "github.com/gorilla/handlers"
|
||||
"log"
|
||||
"math/rand"
|
||||
"mime"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
context "golang.org/x/net/context"
|
||||
|
||||
"github.com/PuerkitoBio/ghost/handlers"
|
||||
"github.com/VojtechVitek/ratelimit"
|
||||
"github.com/VojtechVitek/ratelimit/memory"
|
||||
gorillaHandlers "github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
// import pprof
|
||||
_ "net/http/pprof"
|
||||
|
||||
"crypto/tls"
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
|
||||
web "github.com/dutchcoders/transfer.sh-web"
|
||||
"github.com/dutchcoders/transfer.sh/server/storage"
|
||||
assetfs "github.com/elazarl/go-bindata-assetfs"
|
||||
|
||||
autocert "golang.org/x/crypto/acme/autocert"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// parse request with maximum memory of _24Kilobits
|
||||
@@ -76,6 +72,13 @@ func ClamavHost(s string) OptionFn {
|
||||
}
|
||||
}
|
||||
|
||||
// PerformClamavPrescan enables clamav prescan on upload
|
||||
func PerformClamavPrescan(b bool) OptionFn {
|
||||
return func(srvr *Server) {
|
||||
srvr.performClamavPrescan = b
|
||||
}
|
||||
}
|
||||
|
||||
// VirustotalKey sets virus total key
|
||||
func VirustotalKey(s string) OptionFn {
|
||||
return func(srvr *Server) {
|
||||
@@ -241,7 +244,7 @@ func EnableProfiler() OptionFn {
|
||||
}
|
||||
|
||||
// UseStorage set storage to use
|
||||
func UseStorage(s Storage) OptionFn {
|
||||
func UseStorage(s storage.Storage) OptionFn {
|
||||
return func(srvr *Server) {
|
||||
srvr.storage = s
|
||||
}
|
||||
@@ -330,7 +333,7 @@ type Server struct {
|
||||
purgeDays time.Duration
|
||||
purgeInterval time.Duration
|
||||
|
||||
storage Storage
|
||||
storage storage.Storage
|
||||
|
||||
forceHTTPS bool
|
||||
|
||||
@@ -338,8 +341,9 @@ type Server struct {
|
||||
|
||||
ipFilterOptions *IPFilterOptions
|
||||
|
||||
VirusTotalKey string
|
||||
ClamAVDaemonHost string
|
||||
VirusTotalKey string
|
||||
ClamAVDaemonHost string
|
||||
performClamavPrescan bool
|
||||
|
||||
tempPath string
|
||||
|
||||
@@ -377,7 +381,7 @@ func New(options ...OptionFn) (*Server, error) {
|
||||
|
||||
func init() {
|
||||
var seedBytes [8]byte
|
||||
if _, err := crypto_rand.Read(seedBytes[:]); err != nil {
|
||||
if _, err := cryptoRand.Read(seedBytes[:]); err != nil {
|
||||
panic("cannot obtain cryptographically secure seed")
|
||||
}
|
||||
rand.Seed(int64(binary.LittleEndian.Uint64(seedBytes[:])))
|
||||
@@ -424,13 +428,17 @@ func (s *Server) Run() {
|
||||
s.logger.Panicf("Unable to parse: path=%s, err=%s", path, err)
|
||||
}
|
||||
|
||||
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||
if err != nil {
|
||||
s.logger.Println("Unable to parse html template", err)
|
||||
if strings.HasSuffix(path, ".html") {
|
||||
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||
if err != nil {
|
||||
s.logger.Println("Unable to parse html template", err)
|
||||
}
|
||||
}
|
||||
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||
if err != nil {
|
||||
s.logger.Println("Unable to parse text template", err)
|
||||
if strings.HasSuffix(path, ".txt") {
|
||||
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||
if err != nil {
|
||||
s.logger.Println("Unable to parse text template", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -467,7 +475,7 @@ func (s *Server) Run() {
|
||||
return false
|
||||
}
|
||||
|
||||
match = (r.Referer() == "")
|
||||
match = r.Referer() == ""
|
||||
|
||||
u, err := url.Parse(r.Referer())
|
||||
if err != nil {
|
||||
@@ -529,32 +537,34 @@ func (s *Server) Run() {
|
||||
)
|
||||
|
||||
if !s.TLSListenerOnly {
|
||||
srvr := &http.Server{
|
||||
Addr: s.ListenerString,
|
||||
Handler: h,
|
||||
}
|
||||
|
||||
listening = true
|
||||
s.logger.Printf("listening on port: %v\n", s.ListenerString)
|
||||
s.logger.Printf("starting to listen on: %v\n", s.ListenerString)
|
||||
|
||||
go func() {
|
||||
_ = srvr.ListenAndServe()
|
||||
srvr := &http.Server{
|
||||
Addr: s.ListenerString,
|
||||
Handler: h,
|
||||
}
|
||||
|
||||
if err := srvr.ListenAndServe(); err != nil {
|
||||
s.logger.Fatal(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if s.TLSListenerString != "" {
|
||||
listening = true
|
||||
s.logger.Printf("listening on port: %v\n", s.TLSListenerString)
|
||||
s.logger.Printf("starting to listen for TLS on: %v\n", s.TLSListenerString)
|
||||
|
||||
go func() {
|
||||
s := &http.Server{
|
||||
srvr := &http.Server{
|
||||
Addr: s.TLSListenerString,
|
||||
Handler: h,
|
||||
TLSConfig: s.tlsConfig,
|
||||
}
|
||||
|
||||
if err := s.ListenAndServeTLS("", ""); err != nil {
|
||||
panic(err)
|
||||
if err := srvr.ListenAndServeTLS("", ""); err != nil {
|
||||
s.logger.Fatal(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -1,808 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
"storj.io/common/fpath"
|
||||
"storj.io/common/storj"
|
||||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
// Storage is the interface for storage operation
|
||||
type Storage interface {
|
||||
// Get retrieves a file from storage
|
||||
Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error)
|
||||
// Head retrieves content length of a file from storage
|
||||
Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
|
||||
// Put saves a file on storage
|
||||
Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
|
||||
// Delete removes a file from storage
|
||||
Delete(ctx context.Context, token string, filename string) error
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
IsNotExist(err error) bool
|
||||
// Purge cleans up the storage
|
||||
Purge(ctx context.Context, days time.Duration) error
|
||||
|
||||
// Type returns the storage type
|
||||
Type() string
|
||||
}
|
||||
|
||||
// LocalStorage is a local storage
|
||||
type LocalStorage struct {
|
||||
Storage
|
||||
basedir string
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewLocalStorage is the factory for LocalStorage
|
||||
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
|
||||
return &LocalStorage{basedir: basedir, logger: logger}, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *LocalStorage) Type() string {
|
||||
return "local"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *LocalStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
|
||||
var fi os.FileInfo
|
||||
if fi, err = os.Lstat(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *LocalStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
|
||||
// content type , content length
|
||||
if reader, err = os.Open(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
if fi, err = os.Lstat(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *LocalStorage) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
|
||||
_ = os.Remove(metadata)
|
||||
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
err = os.Remove(path)
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *LocalStorage) Purge(ctx context.Context, days time.Duration) (err error) {
|
||||
err = filepath.Walk(s.basedir,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.ModTime().Before(time.Now().Add(-1 * days)) {
|
||||
err = os.Remove(path)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *LocalStorage) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *LocalStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
|
||||
var f io.WriteCloser
|
||||
var err error
|
||||
|
||||
path := filepath.Join(s.basedir, token)
|
||||
|
||||
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
defer CloseCheck(f.Close)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = io.Copy(f, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// S3Storage is a storage backed by AWS S3
|
||||
type S3Storage struct {
|
||||
Storage
|
||||
bucket string
|
||||
session *session.Session
|
||||
s3 *s3.S3
|
||||
logger *log.Logger
|
||||
purgeDays time.Duration
|
||||
noMultipart bool
|
||||
}
|
||||
|
||||
// NewS3Storage is the factory for S3Storage
|
||||
func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
|
||||
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
|
||||
|
||||
return &S3Storage{
|
||||
bucket: bucketName,
|
||||
s3: s3.New(sess),
|
||||
session: sess,
|
||||
logger: logger,
|
||||
noMultipart: disableMultipart,
|
||||
purgeDays: time.Duration(purgeDays*24) * time.Hour,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *S3Storage) Type() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
headRequest := &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
// content type , content length
|
||||
response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if response.ContentLength != nil {
|
||||
contentLength = uint64(*response.ContentLength)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *S3Storage) Purge(ctx context.Context, days time.Duration) (err error) {
|
||||
// NOOP expiration is set at upload time
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *S3Storage) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
switch aerr.Code() {
|
||||
case s3.ErrCodeNoSuchKey:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *S3Storage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
getRequest := &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
response, err := s.s3.GetObjectWithContext(ctx, getRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if response.ContentLength != nil {
|
||||
contentLength = uint64(*response.ContentLength)
|
||||
}
|
||||
|
||||
reader = response.Body
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
|
||||
deleteRequest := &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(metadata),
|
||||
}
|
||||
|
||||
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
deleteRequest = &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
s.logger.Printf("Uploading file %s to S3 Bucket", filename)
|
||||
var concurrency int
|
||||
if !s.noMultipart {
|
||||
concurrency = 20
|
||||
} else {
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
// Create an uploader with the session and custom options
|
||||
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
|
||||
u.Concurrency = concurrency // default is 5
|
||||
u.LeavePartsOnError = false
|
||||
})
|
||||
|
||||
var expire *time.Time
|
||||
if s.purgeDays.Hours() > 0 {
|
||||
expire = aws.Time(time.Now().Add(s.purgeDays))
|
||||
}
|
||||
|
||||
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: reader,
|
||||
Expires: expire,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GDrive is a storage backed by GDrive
|
||||
type GDrive struct {
|
||||
service *drive.Service
|
||||
rootID string
|
||||
basedir string
|
||||
localConfigPath string
|
||||
chunkSize int
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewGDriveStorage is the factory for GDrive
|
||||
func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
|
||||
b, err := ioutil.ReadFile(clientJSONFilepath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If modifying these scopes, delete your previously saved client_secret.json.
|
||||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// ToDo: Upgrade deprecated version
|
||||
srv, err := drive.New(getGDriveClient(context.TODO(), config, localConfigPath, logger)) // nolint: staticcheck
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chunkSize = chunkSize * 1024 * 1024
|
||||
storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
|
||||
err = storage.setupRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
const gdriveRootConfigFile = "root_id.conf"
|
||||
const gdriveTokenJSONFile = "token.json"
|
||||
const gdriveDirectoryMimeType = "application/vnd.google-apps.folder"
|
||||
|
||||
func (s *GDrive) setupRoot() error {
|
||||
rootFileConfig := filepath.Join(s.localConfigPath, gdriveRootConfigFile)
|
||||
|
||||
rootID, err := ioutil.ReadFile(rootFileConfig)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
if string(rootID) != "" {
|
||||
s.rootID = string(rootID)
|
||||
return nil
|
||||
}
|
||||
|
||||
dir := &drive.File{
|
||||
Name: s.basedir,
|
||||
MimeType: gdriveDirectoryMimeType,
|
||||
}
|
||||
|
||||
di, err := s.service.Files.Create(dir).Fields("id").Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.rootID = di.Id
|
||||
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GDrive) hasChecksum(f *drive.File) bool {
|
||||
return f.Md5Checksum != ""
|
||||
}
|
||||
|
||||
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
|
||||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
|
||||
}
|
||||
|
||||
func (s *GDrive) findID(filename string, token string) (string, error) {
|
||||
filename = strings.Replace(filename, `'`, `\'`, -1)
|
||||
filename = strings.Replace(filename, `"`, `\"`, -1)
|
||||
|
||||
fileID, tokenID, nextPageToken := "", "", ""
|
||||
|
||||
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gdriveDirectoryMimeType)
|
||||
l, err := s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
tokenID = fi.Id
|
||||
break
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if filename == "" {
|
||||
return tokenID, nil
|
||||
} else if tokenID == "" {
|
||||
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
}
|
||||
|
||||
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gdriveDirectoryMimeType)
|
||||
l, err = s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
|
||||
fileID = fi.Id
|
||||
break
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if fileID == "" {
|
||||
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
}
|
||||
|
||||
return fileID, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *GDrive) Type() string {
|
||||
return "gdrive"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var fi *drive.File
|
||||
if fi, err = s.service.Files.Get(fileID).Fields("size").Do(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var fi *drive.File
|
||||
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !s.hasChecksum(fi) {
|
||||
err = fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size)
|
||||
|
||||
var res *http.Response
|
||||
res, err = s.service.Files.Get(fileID).Context(ctx).Download()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
reader = res.Body
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
|
||||
_ = s.service.Files.Delete(metadata).Do()
|
||||
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = s.service.Files.Delete(fileID).Do()
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
|
||||
nextPageToken := ""
|
||||
|
||||
expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
|
||||
q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gdriveDirectoryMimeType)
|
||||
l, err := s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
err = s.service.Files.Delete(fi.Id).Do()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *GDrive) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if e, ok := err.(*googleapi.Error); ok {
|
||||
return e.Code == http.StatusNotFound
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
|
||||
dirID, err := s.findID("", token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dirID == "" {
|
||||
dir := &drive.File{
|
||||
Name: token,
|
||||
Parents: []string{s.rootID},
|
||||
MimeType: gdriveDirectoryMimeType,
|
||||
Size: int64(contentLength),
|
||||
}
|
||||
|
||||
di, err := s.service.Files.Create(dir).Fields("id").Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dirID = di.Id
|
||||
}
|
||||
|
||||
// Instantiate empty drive file
|
||||
dst := &drive.File{
|
||||
Name: filename,
|
||||
Parents: []string{dirID},
|
||||
MimeType: contentType,
|
||||
}
|
||||
|
||||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retrieve a token, saves the token, then returns the generated client.
|
||||
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
|
||||
tokenFile := filepath.Join(localConfigPath, gdriveTokenJSONFile)
|
||||
tok, err := gDriveTokenFromFile(tokenFile)
|
||||
if err != nil {
|
||||
tok = getGDriveTokenFromWeb(ctx, config, logger)
|
||||
saveGDriveToken(tokenFile, tok, logger)
|
||||
}
|
||||
|
||||
return config.Client(ctx, tok)
|
||||
}
|
||||
|
||||
// Request a token from the web, then returns the retrieved token.
|
||||
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
|
||||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
|
||||
fmt.Printf("Go to the following link in your browser then type the "+
|
||||
"authorization code: \n%v\n", authURL)
|
||||
|
||||
var authCode string
|
||||
if _, err := fmt.Scan(&authCode); err != nil {
|
||||
logger.Fatalf("Unable to read authorization code %v", err)
|
||||
}
|
||||
|
||||
tok, err := config.Exchange(ctx, authCode)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to retrieve token from web %v", err)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
|
||||
// Retrieves a token from a local file.
|
||||
func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
|
||||
f, err := os.Open(file)
|
||||
defer CloseCheck(f.Close)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tok := &oauth2.Token{}
|
||||
err = json.NewDecoder(f).Decode(tok)
|
||||
return tok, err
|
||||
}
|
||||
|
||||
// Saves a token to a file path.
|
||||
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
|
||||
logger.Printf("Saving credential file to: %s\n", path)
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
defer CloseCheck(f.Close)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to cache oauth token: %v", err)
|
||||
}
|
||||
|
||||
err = json.NewEncoder(f).Encode(token)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to encode oauth token: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// StorjStorage is a storage backed by Storj
|
||||
type StorjStorage struct {
|
||||
Storage
|
||||
project *uplink.Project
|
||||
bucket *uplink.Bucket
|
||||
purgeDays time.Duration
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewStorjStorage is the factory for StorjStorage
|
||||
func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
|
||||
var instance StorjStorage
|
||||
var err error
|
||||
|
||||
pCtx := context.TODO()
|
||||
|
||||
ctx := fpath.WithTempData(pCtx, "", true)
|
||||
|
||||
parsedAccess, err := uplink.ParseAccess(access)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.project, err = uplink.OpenProject(ctx, parsedAccess)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
|
||||
if err != nil {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = instance.project.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
|
||||
|
||||
instance.logger = logger
|
||||
|
||||
return &instance, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *StorjStorage) Type() string {
|
||||
return "storj"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
contentLength = uint64(obj.System.ContentLength)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *StorjStorage) Get(ctx context.Context, token string, filename string) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Getting file %s from Storj Bucket", filename)
|
||||
|
||||
download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
contentLength = uint64(download.Info().System.ContentLength)
|
||||
|
||||
reader = download
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Deleting file %s from Storj Bucket", filename)
|
||||
|
||||
_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *StorjStorage) Purge(ctx context.Context, days time.Duration) (err error) {
|
||||
// NOOP expiration is set at upload time
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Uploading file %s to Storj Bucket", filename)
|
||||
|
||||
var uploadOptions *uplink.UploadOptions
|
||||
if s.purgeDays.Hours() > 0 {
|
||||
uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
|
||||
}
|
||||
|
||||
writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := io.Copy(writer, reader)
|
||||
if err != nil || uint64(n) != contentLength {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = writer.Abort()
|
||||
return err
|
||||
}
|
||||
err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
|
||||
if err != nil {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = writer.Abort()
|
||||
return err
|
||||
}
|
||||
|
||||
err = writer.Commit()
|
||||
return err
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *StorjStorage) IsNotExist(err error) bool {
|
||||
return errors.Is(err, uplink.ErrObjectNotFound)
|
||||
}
|
||||
120
server/storage/common.go
Normal file
120
server/storage/common.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"regexp"
|
||||
)
|
||||
|
||||
type Range struct {
|
||||
Start uint64
|
||||
Limit uint64
|
||||
contentRange string
|
||||
}
|
||||
|
||||
// Range Reconstructs Range header and returns it
|
||||
func (r *Range) Range() string {
|
||||
if r.Limit > 0 {
|
||||
return fmt.Sprintf("bytes=%d-%d", r.Start, r.Start+r.Limit-1)
|
||||
} else {
|
||||
return fmt.Sprintf("bytes=%d-", r.Start)
|
||||
}
|
||||
}
|
||||
|
||||
// AcceptLength Tries to accept given range
|
||||
// returns newContentLength if range was satisfied, otherwise returns given contentLength
|
||||
func (r *Range) AcceptLength(contentLength uint64) (newContentLength uint64) {
|
||||
newContentLength = contentLength
|
||||
if r.Limit == 0 {
|
||||
r.Limit = newContentLength - r.Start
|
||||
}
|
||||
if contentLength < r.Start {
|
||||
return
|
||||
}
|
||||
if r.Limit > contentLength-r.Start {
|
||||
return
|
||||
}
|
||||
r.contentRange = fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Limit-1, contentLength)
|
||||
newContentLength = r.Limit
|
||||
return
|
||||
}
|
||||
|
||||
func (r *Range) SetContentRange(cr string) {
|
||||
r.contentRange = cr
|
||||
}
|
||||
|
||||
// Returns accepted Content-Range header. If range wasn't accepted empty string is returned
|
||||
func (r *Range) ContentRange() string {
|
||||
return r.contentRange
|
||||
}
|
||||
|
||||
var rexp *regexp.Regexp = regexp.MustCompile(`^bytes=([0-9]+)-([0-9]*)$`)
|
||||
|
||||
// Parses HTTP Range header and returns struct on success
|
||||
// only bytes=start-finish supported
|
||||
func ParseRange(rng string) *Range {
|
||||
if rng == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
matches := rexp.FindAllStringSubmatch(rng, -1)
|
||||
if len(matches) != 1 || len(matches[0]) != 3 {
|
||||
return nil
|
||||
}
|
||||
if len(matches[0][0]) != len(rng) || len(matches[0][1]) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
start, err := strconv.ParseUint(matches[0][1], 10, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(matches[0][2]) == 0 {
|
||||
return &Range{Start: start, Limit: 0}
|
||||
}
|
||||
|
||||
finish, err := strconv.ParseUint(matches[0][2], 10, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if finish < start || finish+1 < finish {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Range{Start: start, Limit: finish - start + 1}
|
||||
}
|
||||
|
||||
// Storage is the interface for storage operation
|
||||
type Storage interface {
|
||||
// Get retrieves a file from storage
|
||||
Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error)
|
||||
// Head retrieves content length of a file from storage
|
||||
Head(ctx context.Context, token string, filename string) (contentLength uint64, err error)
|
||||
// Put saves a file on storage
|
||||
Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
|
||||
// Delete removes a file from storage
|
||||
Delete(ctx context.Context, token string, filename string) error
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
IsNotExist(err error) bool
|
||||
// Purge cleans up the storage
|
||||
Purge(ctx context.Context, days time.Duration) error
|
||||
// Whether storage supports Get with Range header
|
||||
IsRangeSupported() bool
|
||||
// Type returns the storage type
|
||||
Type() string
|
||||
}
|
||||
|
||||
func CloseCheck(c io.Closer) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := c.Close(); err != nil {
|
||||
fmt.Println("Received close error:", err)
|
||||
}
|
||||
}
|
||||
394
server/storage/gdrive.go
Normal file
394
server/storage/gdrive.go
Normal file
@@ -0,0 +1,394 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// GDrive is a storage backed by GDrive
|
||||
type GDrive struct {
|
||||
service *drive.Service
|
||||
rootID string
|
||||
basedir string
|
||||
localConfigPath string
|
||||
chunkSize int
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
const gDriveRootConfigFile = "root_id.conf"
|
||||
const gDriveTokenJSONFile = "token.json"
|
||||
const gDriveDirectoryMimeType = "application/vnd.google-apps.folder"
|
||||
|
||||
// NewGDriveStorage is the factory for GDrive
|
||||
func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) {
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
b, err := ioutil.ReadFile(clientJSONFilepath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If modifying these scopes, delete your previously saved client_secret.json.
|
||||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpClient := getGDriveClient(ctx, config, localConfigPath, logger)
|
||||
|
||||
srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
|
||||
err = storage.setupRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (s *GDrive) setupRoot() error {
|
||||
rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile)
|
||||
|
||||
rootID, err := ioutil.ReadFile(rootFileConfig)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
if string(rootID) != "" {
|
||||
s.rootID = string(rootID)
|
||||
return nil
|
||||
}
|
||||
|
||||
dir := &drive.File{
|
||||
Name: s.basedir,
|
||||
MimeType: gDriveDirectoryMimeType,
|
||||
}
|
||||
|
||||
di, err := s.service.Files.Create(dir).Fields("id").Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.rootID = di.Id
|
||||
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GDrive) hasChecksum(f *drive.File) bool {
|
||||
return f.Md5Checksum != ""
|
||||
}
|
||||
|
||||
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
|
||||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
|
||||
}
|
||||
|
||||
func (s *GDrive) findID(filename string, token string) (string, error) {
|
||||
filename = strings.Replace(filename, `'`, `\'`, -1)
|
||||
filename = strings.Replace(filename, `"`, `\"`, -1)
|
||||
|
||||
fileID, tokenID, nextPageToken := "", "", ""
|
||||
|
||||
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootID, token, gDriveDirectoryMimeType)
|
||||
l, err := s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
tokenID = fi.Id
|
||||
break
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if filename == "" {
|
||||
return tokenID, nil
|
||||
} else if tokenID == "" {
|
||||
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
}
|
||||
|
||||
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenID, filename, gDriveDirectoryMimeType)
|
||||
l, err = s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
|
||||
fileID = fi.Id
|
||||
break
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if fileID == "" {
|
||||
return "", fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
}
|
||||
|
||||
return fileID, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *GDrive) Type() string {
|
||||
return "gdrive"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *GDrive) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var fi *drive.File
|
||||
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *GDrive) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var fi *drive.File
|
||||
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !s.hasChecksum(fi) {
|
||||
err = fmt.Errorf("cannot find file %s/%s", token, filename)
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size)
|
||||
|
||||
fileGetCall := s.service.Files.Get(fileID)
|
||||
if rng != nil {
|
||||
header := fileGetCall.Header()
|
||||
header.Set("Range", rng.Range())
|
||||
}
|
||||
|
||||
var res *http.Response
|
||||
res, err = fileGetCall.Context(ctx).Download()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if rng != nil {
|
||||
reader = res.Body
|
||||
rng.AcceptLength(contentLength)
|
||||
return
|
||||
}
|
||||
|
||||
reader = res.Body
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
|
||||
_ = s.service.Files.Delete(metadata).Do()
|
||||
|
||||
var fileID string
|
||||
fileID, err = s.findID(filename, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = s.service.Files.Delete(fileID).Context(ctx).Do()
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {
|
||||
nextPageToken := ""
|
||||
|
||||
expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
|
||||
q := fmt.Sprintf("'%s' in parents and modifiedTime < '%s' and mimeType!='%s' and trashed=false", s.rootID, expirationDate, gDriveDirectoryMimeType)
|
||||
l, err := s.list(nextPageToken, q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for 0 < len(l.Files) {
|
||||
for _, fi := range l.Files {
|
||||
err = s.service.Files.Delete(fi.Id).Context(ctx).Do()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if l.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
l, err = s.list(l.NextPageToken, q)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *GDrive) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if e, ok := err.(*googleapi.Error); ok {
|
||||
return e.Code == http.StatusNotFound
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *GDrive) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
|
||||
dirID, err := s.findID("", token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dirID == "" {
|
||||
dir := &drive.File{
|
||||
Name: token,
|
||||
Parents: []string{s.rootID},
|
||||
MimeType: gDriveDirectoryMimeType,
|
||||
}
|
||||
|
||||
di, err := s.service.Files.Create(dir).Fields("id").Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dirID = di.Id
|
||||
}
|
||||
|
||||
// Instantiate empty drive file
|
||||
dst := &drive.File{
|
||||
Name: filename,
|
||||
Parents: []string{dirID},
|
||||
MimeType: contentType,
|
||||
}
|
||||
|
||||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GDrive) IsRangeSupported() bool { return true }
|
||||
|
||||
// Retrieve a token, saves the token, then returns the generated client.
|
||||
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
|
||||
tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile)
|
||||
tok, err := gDriveTokenFromFile(tokenFile)
|
||||
if err != nil {
|
||||
tok = getGDriveTokenFromWeb(ctx, config, logger)
|
||||
saveGDriveToken(tokenFile, tok, logger)
|
||||
}
|
||||
|
||||
return config.Client(ctx, tok)
|
||||
}
|
||||
|
||||
// Request a token from the web, then returns the retrieved token.
|
||||
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
|
||||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
|
||||
fmt.Printf("Go to the following link in your browser then type the "+
|
||||
"authorization code: \n%v\n", authURL)
|
||||
|
||||
var authCode string
|
||||
if _, err := fmt.Scan(&authCode); err != nil {
|
||||
logger.Fatalf("Unable to read authorization code %v", err)
|
||||
}
|
||||
|
||||
tok, err := config.Exchange(ctx, authCode)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to retrieve token from web %v", err)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
|
||||
// Retrieves a token from a local file.
|
||||
func gDriveTokenFromFile(file string) (*oauth2.Token, error) {
|
||||
f, err := os.Open(file)
|
||||
defer CloseCheck(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tok := &oauth2.Token{}
|
||||
err = json.NewDecoder(f).Decode(tok)
|
||||
return tok, err
|
||||
}
|
||||
|
||||
// Saves a token to a file path.
|
||||
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) {
|
||||
logger.Printf("Saving credential file to: %s\n", path)
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
defer CloseCheck(f)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to cache oauth token: %v", err)
|
||||
}
|
||||
|
||||
err = json.NewEncoder(f).Encode(token)
|
||||
if err != nil {
|
||||
logger.Fatalf("Unable to encode oauth token: %v", err)
|
||||
}
|
||||
}
|
||||
138
server/storage/local.go
Normal file
138
server/storage/local.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LocalStorage is a local storage
|
||||
type LocalStorage struct {
|
||||
Storage
|
||||
basedir string
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewLocalStorage is the factory for LocalStorage
|
||||
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) {
|
||||
return &LocalStorage{basedir: basedir, logger: logger}, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *LocalStorage) Type() string {
|
||||
return "local"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *LocalStorage) Head(_ context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
|
||||
var fi os.FileInfo
|
||||
if fi, err = os.Lstat(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *LocalStorage) Get(_ context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
|
||||
var file *os.File
|
||||
|
||||
// content type , content length
|
||||
if file, err = os.Open(path); err != nil {
|
||||
return
|
||||
}
|
||||
reader = file
|
||||
|
||||
var fi os.FileInfo
|
||||
if fi, err = os.Lstat(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
contentLength = uint64(fi.Size())
|
||||
if rng != nil {
|
||||
contentLength = rng.AcceptLength(contentLength)
|
||||
if _, err = file.Seek(int64(rng.Start), 0); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *LocalStorage) Delete(_ context.Context, token string, filename string) (err error) {
|
||||
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename))
|
||||
_ = os.Remove(metadata)
|
||||
|
||||
path := filepath.Join(s.basedir, token, filename)
|
||||
err = os.Remove(path)
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *LocalStorage) Purge(_ context.Context, days time.Duration) (err error) {
|
||||
err = filepath.Walk(s.basedir,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.ModTime().Before(time.Now().Add(-1 * days)) {
|
||||
err = os.Remove(path)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *LocalStorage) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *LocalStorage) Put(_ context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) error {
|
||||
var f io.WriteCloser
|
||||
var err error
|
||||
|
||||
path := filepath.Join(s.basedir, token)
|
||||
|
||||
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
defer CloseCheck(f)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = io.Copy(f, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LocalStorage) IsRangeSupported() bool { return true }
|
||||
188
server/storage/s3.go
Normal file
188
server/storage/s3.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// S3Storage is a storage backed by AWS S3
|
||||
type S3Storage struct {
|
||||
Storage
|
||||
bucket string
|
||||
session *session.Session
|
||||
s3 *s3.S3
|
||||
logger *log.Logger
|
||||
purgeDays time.Duration
|
||||
noMultipart bool
|
||||
}
|
||||
|
||||
// NewS3Storage is the factory for S3Storage
|
||||
func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
|
||||
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
|
||||
|
||||
return &S3Storage{
|
||||
bucket: bucketName,
|
||||
s3: s3.New(sess),
|
||||
session: sess,
|
||||
logger: logger,
|
||||
noMultipart: disableMultipart,
|
||||
purgeDays: time.Duration(purgeDays*24) * time.Hour,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *S3Storage) Type() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *S3Storage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
headRequest := &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
// content type , content length
|
||||
response, err := s.s3.HeadObjectWithContext(ctx, headRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if response.ContentLength != nil {
|
||||
contentLength = uint64(*response.ContentLength)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *S3Storage) Purge(context.Context, time.Duration) (err error) {
|
||||
// NOOP expiration is set at upload time
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *S3Storage) IsNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
switch aerr.Code() {
|
||||
case s3.ErrCodeNoSuchKey:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *S3Storage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
getRequest := &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
if rng != nil {
|
||||
getRequest.Range = aws.String(rng.Range())
|
||||
}
|
||||
|
||||
response, err := s.s3.GetObjectWithContext(ctx, getRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if response.ContentLength != nil {
|
||||
contentLength = uint64(*response.ContentLength)
|
||||
}
|
||||
if rng != nil && response.ContentRange != nil {
|
||||
rng.SetContentRange(*response.ContentRange)
|
||||
}
|
||||
|
||||
reader = response.Body
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *S3Storage) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
metadata := fmt.Sprintf("%s/%s.metadata", token, filename)
|
||||
deleteRequest := &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(metadata),
|
||||
}
|
||||
|
||||
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
deleteRequest = &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}
|
||||
|
||||
_, err = s.s3.DeleteObjectWithContext(ctx, deleteRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *S3Storage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, _ uint64) (err error) {
|
||||
key := fmt.Sprintf("%s/%s", token, filename)
|
||||
|
||||
s.logger.Printf("Uploading file %s to S3 Bucket", filename)
|
||||
var concurrency int
|
||||
if !s.noMultipart {
|
||||
concurrency = 20
|
||||
} else {
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
// Create an uploader with the session and custom options
|
||||
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
|
||||
u.Concurrency = concurrency // default is 5
|
||||
u.LeavePartsOnError = false
|
||||
})
|
||||
|
||||
var expire *time.Time
|
||||
if s.purgeDays.Hours() > 0 {
|
||||
expire = aws.Time(time.Now().Add(s.purgeDays))
|
||||
}
|
||||
|
||||
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: reader,
|
||||
Expires: expire,
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *S3Storage) IsRangeSupported() bool { return true }
|
||||
|
||||
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
|
||||
return session.Must(session.NewSession(&aws.Config{
|
||||
Region: aws.String(region),
|
||||
Endpoint: aws.String(endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
S3ForcePathStyle: aws.Bool(forcePathStyle),
|
||||
}))
|
||||
}
|
||||
163
server/storage/storj.go
Normal file
163
server/storage/storj.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"storj.io/common/fpath"
|
||||
"storj.io/common/storj"
|
||||
"storj.io/uplink"
|
||||
)
|
||||
|
||||
// StorjStorage is a storage backed by Storj
|
||||
type StorjStorage struct {
|
||||
Storage
|
||||
project *uplink.Project
|
||||
bucket *uplink.Bucket
|
||||
purgeDays time.Duration
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewStorjStorage is the factory for StorjStorage
|
||||
func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
|
||||
var instance StorjStorage
|
||||
var err error
|
||||
|
||||
pCtx := context.TODO()
|
||||
|
||||
ctx := fpath.WithTempData(pCtx, "", true)
|
||||
|
||||
uplConf := &uplink.Config{
|
||||
UserAgent: "transfer-sh",
|
||||
}
|
||||
|
||||
parsedAccess, err := uplink.ParseAccess(access)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.project, err = uplConf.OpenProject(ctx, parsedAccess)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.bucket, err = instance.project.EnsureBucket(ctx, bucket)
|
||||
if err != nil {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = instance.project.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
instance.purgeDays = time.Duration(purgeDays*24) * time.Hour
|
||||
|
||||
instance.logger = logger
|
||||
|
||||
return &instance, nil
|
||||
}
|
||||
|
||||
// Type returns the storage type
|
||||
func (s *StorjStorage) Type() string {
|
||||
return "storj"
|
||||
}
|
||||
|
||||
// Head retrieves content length of a file from storage
|
||||
func (s *StorjStorage) Head(ctx context.Context, token string, filename string) (contentLength uint64, err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
obj, err := s.project.StatObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
contentLength = uint64(obj.System.ContentLength)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Get retrieves a file from storage
|
||||
func (s *StorjStorage) Get(ctx context.Context, token string, filename string, rng *Range) (reader io.ReadCloser, contentLength uint64, err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Getting file %s from Storj Bucket", filename)
|
||||
|
||||
options := uplink.DownloadOptions{}
|
||||
if rng != nil {
|
||||
options.Offset = int64(rng.Start)
|
||||
if rng.Limit > 0 {
|
||||
options.Length = int64(rng.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
download, err := s.project.DownloadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, &options)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
contentLength = uint64(download.Info().System.ContentLength)
|
||||
if rng != nil {
|
||||
contentLength = rng.AcceptLength(contentLength)
|
||||
}
|
||||
|
||||
reader = download
|
||||
return
|
||||
}
|
||||
|
||||
// Delete removes a file from storage
|
||||
func (s *StorjStorage) Delete(ctx context.Context, token string, filename string) (err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Deleting file %s from Storj Bucket", filename)
|
||||
|
||||
_, err = s.project.DeleteObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Purge cleans up the storage
|
||||
func (s *StorjStorage) Purge(context.Context, time.Duration) (err error) {
|
||||
// NOOP expiration is set at upload time
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put saves a file on storage
|
||||
func (s *StorjStorage) Put(ctx context.Context, token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
|
||||
key := storj.JoinPaths(token, filename)
|
||||
|
||||
s.logger.Printf("Uploading file %s to Storj Bucket", filename)
|
||||
|
||||
var uploadOptions *uplink.UploadOptions
|
||||
if s.purgeDays.Hours() > 0 {
|
||||
uploadOptions = &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)}
|
||||
}
|
||||
|
||||
writer, err := s.project.UploadObject(fpath.WithTempData(ctx, "", true), s.bucket.Name, key, uploadOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := io.Copy(writer, reader)
|
||||
if err != nil || uint64(n) != contentLength {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = writer.Abort()
|
||||
return err
|
||||
}
|
||||
err = writer.SetCustomMetadata(ctx, uplink.CustomMetadata{"content-type": contentType})
|
||||
if err != nil {
|
||||
//Ignoring the error to return the one that occurred first, but try to clean up.
|
||||
_ = writer.Abort()
|
||||
return err
|
||||
}
|
||||
|
||||
err = writer.Commit()
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *StorjStorage) IsRangeSupported() bool { return true }
|
||||
|
||||
// IsNotExist indicates if a file doesn't exist on storage
|
||||
func (s *StorjStorage) IsNotExist(err error) bool {
|
||||
return errors.Is(err, uplink.ErrObjectNotFound)
|
||||
}
|
||||
@@ -30,25 +30,12 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/mail"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/golang/gddo/httputil/header"
|
||||
)
|
||||
|
||||
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
|
||||
return session.Must(session.NewSession(&aws.Config{
|
||||
Region: aws.String(region),
|
||||
Endpoint: aws.String(endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
S3ForcePathStyle: aws.Bool(forcePathStyle),
|
||||
}))
|
||||
}
|
||||
|
||||
func formatNumber(format string, s uint64) string {
|
||||
return renderFloat(format, float64(s))
|
||||
}
|
||||
@@ -201,10 +188,6 @@ func renderFloat(format string, n float64) string {
|
||||
return signStr + intStr + decimalStr + fracStr
|
||||
}
|
||||
|
||||
func renderInteger(format string, n int) string {
|
||||
return renderFloat(format, float64(n))
|
||||
}
|
||||
|
||||
// Request.RemoteAddress contains port, which we want to remove i.e.:
|
||||
// "[::1]:58292" => "[::1]"
|
||||
func ipAddrFromRemoteAddr(s string) string {
|
||||
@@ -215,45 +198,16 @@ func ipAddrFromRemoteAddr(s string) string {
|
||||
return s[:idx]
|
||||
}
|
||||
|
||||
func getIPAddress(r *http.Request) string {
|
||||
hdr := r.Header
|
||||
hdrRealIP := hdr.Get("X-Real-Ip")
|
||||
hdrForwardedFor := hdr.Get("X-Forwarded-For")
|
||||
if hdrRealIP == "" && hdrForwardedFor == "" {
|
||||
return ipAddrFromRemoteAddr(r.RemoteAddr)
|
||||
}
|
||||
if hdrForwardedFor != "" {
|
||||
// X-Forwarded-For is potentially a list of addresses separated with ","
|
||||
parts := strings.Split(hdrForwardedFor, ",")
|
||||
for i, p := range parts {
|
||||
parts[i] = strings.TrimSpace(p)
|
||||
}
|
||||
|
||||
// TODO: should return first non-local address
|
||||
return parts[0]
|
||||
}
|
||||
return hdrRealIP
|
||||
}
|
||||
|
||||
func encodeRFC2047(s string) string {
|
||||
// use mail's rfc2047 to encode any string
|
||||
addr := mail.Address{
|
||||
Name: s,
|
||||
Address: "",
|
||||
}
|
||||
return strings.Trim(addr.String(), " <>")
|
||||
}
|
||||
|
||||
func acceptsHTML(hdr http.Header) bool {
|
||||
actual := header.ParseAccept(hdr, "Accept")
|
||||
|
||||
for _, s := range actual {
|
||||
if s.Value == "text/html" {
|
||||
return (true)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return (false)
|
||||
return false
|
||||
}
|
||||
|
||||
func formatSize(size int64) string {
|
||||
@@ -279,9 +233,3 @@ func formatSize(size int64) string {
|
||||
getSuffix := suffixes[int(math.Floor(base))]
|
||||
return fmt.Sprintf("%s %s", strconv.FormatFloat(newVal, 'f', -1, 64), getSuffix)
|
||||
}
|
||||
|
||||
func CloseCheck(f func() error) {
|
||||
if err := f(); err != nil {
|
||||
fmt.Println("Received close error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
virustotal "github.com/dutchcoders/go-virustotal"
|
||||
"github.com/dutchcoders/go-virustotal"
|
||||
)
|
||||
|
||||
func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -45,14 +45,14 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
reader := r.Body
|
||||
|
||||
result, err := vt.Scan(filename, reader)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
s.logger.Println(result)
|
||||
|
||||
Reference in New Issue
Block a user