generated from QuantConnect/Lean.DataSource.SDK
-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathsync.sh
More file actions
40 lines (31 loc) · 1.77 KB
/
sync.sh
File metadata and controls
40 lines (31 loc) · 1.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/bin/bash
processing_date="${QC_DATAFLEET_DEPLOYMENT_DATE}"
processing_date_yesterday="$(date -d "${processing_date} -1 days" +%Y%m%d)"
market="*"
include="*"
filter="_SPOT*"
if [ -n "$2" ]; then
filter="${2}*"
fi
if [ -n "$1" ]; then
market=$(echo "${1}" | tr [a-z] [A-Z])
include="*${market}${filter}"
echo "Downloading crypto TAQ for market: ${market} include ${include}"
else
echo "Downloading crypto TAQ for all markets"
fi
# We store AWS creds and set the CoinAPI creds instead
aws_access_key_id="${AWS_ACCESS_KEY_ID}"
aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}"
export AWS_ACCESS_KEY_ID="${COINAPI_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${COINAPI_AWS_SECRET_ACCESS_KEY}"
# multipart corrupts files
aws configure set default.s3.multipart_threshold 100GB
# We sync yesterdays file too because midnight data of 'processing_date' might be in the end of yesterdays file
aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date_yesterday/$market/ /raw/crypto/coinapi/trades/$processing_date_yesterday/ --exclude='*' --include="${include}"
aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date_yesterday/$market/ /raw/crypto/coinapi/quotes/$processing_date_yesterday/ --exclude='*' --include="${include}"
aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date/$market/ /raw/crypto/coinapi/trades/$processing_date/ --exclude='*' --include="${include}"
aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date/$market/ /raw/crypto/coinapi/quotes/$processing_date/ --exclude='*' --include="${include}"
# Restore AWS creds
export AWS_ACCESS_KEY_ID="$aws_access_key_id"
export AWS_SECRET_ACCESS_KEY="$aws_secret_access_key"