-
Notifications
You must be signed in to change notification settings - Fork 139
Expand file tree
/
Copy pathgenerateGCSSignedUrls.sh
More file actions
108 lines (88 loc) · 3.77 KB
/
Copy pathgenerateGCSSignedUrls.sh
File metadata and controls
108 lines (88 loc) · 3.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/bin/bash
# Script to generate GCS signed URLs for image files in JSONL format
# Usage: ./listgcs.sh <gcs-path> [output-file] [expiration-seconds] [parallel-jobs]
set -e
# Check if GCS path is provided
if [ -z "$1" ]; then
echo "Error: GCS path is required"
echo "Usage: $0 <gcs-path> [output-file] [expiration-seconds] [parallel-jobs]"
echo "Example: $0 gs://my-bucket/images/ output.jsonl 21600 8"
exit 1
fi
GCS_PATH="$1"
OUTPUT_FILE="${2:-signed_urls.jsonl}"
EXPIRATION_SECONDS="${3:-21600}" # Default: 6 hours
PARALLEL_JOBS="${4:-20}" # Default: 20 parallel jobs
# Remove trailing slash from GCS path if present
GCS_PATH="${GCS_PATH%/}"
# Convert seconds to duration format for gcloud (e.g., 21600s)
EXPIRATION="${EXPIRATION_SECONDS}s"
# Image file extensions to include (regex pattern for grep)
IMAGE_PATTERN='\.(jpg|jpeg|png|gif|bmp|webp|tiff|tif|svg)$'
# Function to find an appropriate service account
find_service_account() {
# First, try to get the default compute service account for the current project
local project_id=$(gcloud config get-value project 2>/dev/null)
if [ -n "$project_id" ]; then
local compute_sa="${project_id}-compute@developer.gserviceaccount.com"
if gcloud iam service-accounts describe "$compute_sa" >/dev/null 2>&1; then
echo "$compute_sa"
return 0
fi
fi
# If that doesn't work, try to find any service account in the project
local sa_list=$(gcloud iam service-accounts list --format="value(email)" --limit=1 2>/dev/null)
if [ -n "$sa_list" ]; then
echo "$sa_list" | head -n 1
return 0
fi
return 1
}
# Try to find a service account to use
SERVICE_ACCOUNT=$(find_service_account)
if [ -z "$SERVICE_ACCOUNT" ]; then
echo "Warning: No service account found. Attempting to sign URLs without impersonation."
echo "If this fails, you may need to:"
echo "1. Authenticate with a service account: gcloud auth activate-service-account --key-file=key.json"
echo "2. Or ensure you have appropriate service accounts in your project"
echo ""
fi
# Function to process a single file
process_file() {
local object="$1"
local service_account="$2"
local expiration="$3"
# Create signed URL using gcloud storage sign-url
local signed_url_output
if [ -n "$service_account" ]; then
signed_url_output=$(gcloud storage sign-url --http-verb=GET --duration="$expiration" --impersonate-service-account="$service_account" "$object" 2>/dev/null)
else
signed_url_output=$(gcloud storage sign-url --http-verb=GET --duration="$expiration" "$object" 2>/dev/null)
fi
if [ $? -eq 0 ] && [ -n "$signed_url_output" ]; then
# Extract just the signed_url from the YAML output
local signed_url=$(echo "$signed_url_output" | grep "signed_url:" | sed 's/signed_url: //')
if [ -n "$signed_url" ]; then
# Extract the path after the bucket name and convert slashes to double underscores
local path_part=$(echo "$object" | sed 's|gs://[^/]*/||')
local name_with_path=$(echo "$path_part" | sed 's|/|__|g')
# Output JSONL
echo "{\"name\": \"$name_with_path\", \"url\": \"$signed_url\"}"
fi
fi
}
# Export function and variables for xargs
export -f process_file
export SERVICE_ACCOUNT
export EXPIRATION
echo "Listing files from $GCS_PATH..."
# Get list of all files, filter for images, and process in parallel
gsutil ls -r "$GCS_PATH" 2>/dev/null | \
grep -v '/$' | \
grep -v ':$' | \
grep -iE "$IMAGE_PATTERN" | \
xargs -I {} -P "$PARALLEL_JOBS" bash -c 'process_file "$@"' _ {} "$SERVICE_ACCOUNT" "$EXPIRATION" | \
tee "$OUTPUT_FILE"
echo ""
echo "Done! Signed URLs written to $OUTPUT_FILE"
echo "Total images processed: $(wc -l < "$OUTPUT_FILE")"