Skip to content

Commit f4356d9

Browse files
kukharievkukhariev
committed
fix: implement cloud file operations
1 parent 22c2955 commit f4356d9

4 files changed

Lines changed: 144 additions & 20 deletions

File tree

‎examples/express-gcs.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ const app = express();
55

66
const storage = new GCStorage({ maxUploadSize: '1GB' });
77

8-
storage.onComplete = ({ uri, id }) => {
9-
console.log(`File upload complete, storage path: ${uri}`);
10-
// send gcs link to client
11-
return { id, link: uri };
8+
storage.onComplete = async file => {
9+
const info = await file.get().catch(console.error);
10+
console.log(info);
11+
return file.move(file.originalName).catch(console.error);
1212
};
1313

1414
app.use('/files', uploadx({ storage }));

‎packages/gcs/src/gcs-storage.ts‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as http from 'http';
2121
import request from 'node-fetch';
2222
import { authScopes, BUCKET_NAME, storageAPI, uploadAPI } from './constants';
2323
import { GCSMetaStorage, GCSMetaStorageOptions } from './gcs-meta-storage';
24+
import { resolve } from 'url';
2425

2526
export interface ClientError extends Error {
2627
code: string;
@@ -72,9 +73,13 @@ export interface GCStorageOptions extends BaseStorageOptions<GCSFile>, GoogleAut
7273
metaStorageConfig?: LocalMetaStorageOptions | GCSMetaStorageOptions;
7374
}
7475

75-
export class GCSFile extends File {
76+
export interface GCSFile extends File {
7677
GCSUploadURI?: string;
77-
uri = '';
78+
uri: string;
79+
move: (dest: string) => Promise<Record<string, string>>;
80+
copy: (dest: string) => Promise<Record<string, string>>;
81+
get: () => Promise<Record<string, string>>;
82+
delete: () => Promise<any>;
7883
}
7984

8085
/**
@@ -95,6 +100,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
95100
storageBaseURI: string;
96101
uploadBaseURI: string;
97102
meta: MetaStorage<GCSFile>;
103+
private readonly bucket: string;
98104

99105
constructor(public config: GCStorageOptions = {}) {
100106
super(config);
@@ -109,11 +115,11 @@ export class GCStorage extends BaseStorage<GCSFile> {
109115
}
110116
config.scopes ||= authScopes;
111117
config.keyFile ||= process.env.GCS_KEYFILE;
112-
const bucketName = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
113-
this.storageBaseURI = [storageAPI, bucketName, 'o'].join('/');
114-
this.uploadBaseURI = [uploadAPI, bucketName, 'o'].join('/');
118+
this.bucket = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
119+
this.storageBaseURI = [storageAPI, this.bucket, 'o'].join('/');
120+
this.uploadBaseURI = [uploadAPI, this.bucket, 'o'].join('/');
115121
this.authClient = new GoogleAuth(config);
116-
this._checkBucket(bucketName);
122+
this._checkBucket();
117123
}
118124

119125
normalizeError(error: ClientError): HttpError {
@@ -131,7 +137,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
131137
}
132138

133139
async create(req: http.IncomingMessage, config: FileInit): Promise<GCSFile> {
134-
const file = new GCSFile(config);
140+
const file = new File(config) as GCSFile;
135141
file.name = this.namingFunction(file);
136142
await this.validate(file);
137143
try {
@@ -173,6 +179,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
173179
if (isCompleted(file)) {
174180
file.uri = `${this.storageBaseURI}/${file.name}`;
175181
await this._onComplete(file);
182+
return this.buildCompletedFile(file);
176183
}
177184
return file;
178185
}
@@ -190,6 +197,61 @@ export class GCStorage extends BaseStorage<GCSFile> {
190197
return [{ name } as GCSFile];
191198
}
192199

200+
async copy(name: string, dest: string): Promise<Record<string, string>> {
201+
type CopyProgress = {
202+
rewriteToken?: string;
203+
kind: string;
204+
objectSize: number;
205+
totalBytesRewritten: number;
206+
done: boolean;
207+
resource: Record<string, any>;
208+
};
209+
const newPath = resolve(`/${this.bucket}/${name}`, encodeURI(dest));
210+
const [, bucket, ...pathSegments] = newPath.split('/');
211+
const filename = pathSegments.join('/');
212+
const url = `${this.storageBaseURI}/${name}/rewriteTo/b/${bucket}/o/${filename}`;
213+
let progress = {} as CopyProgress;
214+
const opts = {
215+
body: '',
216+
headers: { 'Content-Type': 'application/json' },
217+
method: 'POST' as const,
218+
url
219+
};
220+
do {
221+
opts.body = progress.rewriteToken
222+
? JSON.stringify({ rewriteToken: progress.rewriteToken })
223+
: '';
224+
progress = (await this.authClient.request<CopyProgress>(opts)).data;
225+
} while (progress.rewriteToken);
226+
return progress.resource;
227+
}
228+
229+
async move(name: string, dest: string): Promise<Record<string, string>> {
230+
const resource = await this.copy(name, dest);
231+
const url = `${this.storageBaseURI}/${name}`;
232+
await this.authClient.request({ method: 'DELETE' as const, url });
233+
return resource;
234+
}
235+
236+
async _get(name: string): Promise<Record<string, string>> {
237+
const url = `${this.storageBaseURI}/${name}`;
238+
return (await this.authClient.request<Record<string, string>>({ url })).data;
239+
}
240+
241+
buildCompletedFile(file: GCSFile): GCSFile {
242+
const completed = { ...file };
243+
completed.lock = async lockFn => {
244+
completed.lockedBy = lockFn;
245+
return Promise.resolve(completed.lockedBy);
246+
};
247+
completed.get = () => this._get(file.name);
248+
completed.delete = () => this.delete(file.name);
249+
completed.copy = async (dest: string) => this.copy(file.name, dest);
250+
completed.move = async (dest: string) => this.move(file.name, dest);
251+
252+
return completed;
253+
}
254+
193255
protected async _write(part: FilePart & GCSFile): Promise<number> {
194256
const { size, uri, body } = part;
195257
const contentRange = buildContentRange(part);
@@ -228,9 +290,9 @@ export class GCStorage extends BaseStorage<GCSFile> {
228290
return this.deleteMeta(file.name);
229291
};
230292

231-
private _checkBucket(bucketName: string): void {
293+
private _checkBucket(): void {
232294
this.authClient
233-
.request({ url: `${storageAPI}/${bucketName}` })
295+
.request({ url: this.storageBaseURI })
234296
.then(() => (this.isReady = true))
235297
.catch((err: ClientError) => {
236298
// eslint-disable-next-line no-console

‎packages/s3/src/s3-storage.ts‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,13 @@ import {
22
AbortMultipartUploadCommand,
33
CompleteMultipartUploadCommand,
44
CompleteMultipartUploadOutput,
5+
CopyObjectCommand,
6+
CopyObjectCommandInput,
7+
CopyObjectCommandOutput,
58
CreateMultipartUploadCommand,
69
CreateMultipartUploadRequest,
10+
DeleteObjectCommand,
11+
DeleteObjectCommandInput,
712
HeadBucketCommand,
813
ListMultipartUploadsCommand,
914
ListPartsCommand,
@@ -34,16 +39,17 @@ import {
3439
import * as http from 'http';
3540
import { AWSError } from './aws-error';
3641
import { S3MetaStorage, S3MetaStorageOptions } from './s3-meta-storage';
42+
import { resolve } from 'url';
3743

3844
const BUCKET_NAME = 'node-uploadx';
3945

4046
export interface S3File extends File {
4147
Parts?: Part[];
4248
UploadId?: string;
4349
uri?: string;
44-
lock: (lockFn: () => any) => Promise<any>;
45-
move: (dest: any) => Promise<any>;
46-
copy: (dest: any) => Promise<any>;
50+
move: (dest: string) => Promise<Record<string, any>>;
51+
copy: (dest: string) => Promise<Record<string, any>>;
52+
get: () => Promise<Record<string, any>>;
4753
delete: () => Promise<any>;
4854
}
4955

@@ -183,6 +189,7 @@ export class S3Storage extends BaseStorage<S3File> {
183189
const [completed] = await this._onComplete(file);
184190
delete file.Parts;
185191
file.uri = completed.Location;
192+
return this.buildCompletedFile(file);
186193
}
187194
return file;
188195
}
@@ -197,6 +204,35 @@ export class S3Storage extends BaseStorage<S3File> {
197204
return [{ name } as S3File];
198205
}
199206

207+
async copy(name: string, dest: string): Promise<CopyObjectCommandOutput> {
208+
const CopySource = `${this.bucket}/${name}`;
209+
const newPath = decodeURI(resolve(`/${CopySource}`, dest)); // path.resolve?
210+
const [, Bucket, ...pathSegments] = newPath.split('/');
211+
const Key = pathSegments.join('/');
212+
const params: CopyObjectCommandInput = { Bucket, Key, CopySource };
213+
return this.client.send(new CopyObjectCommand(params));
214+
}
215+
216+
async move(name: string, dest: string): Promise<CopyObjectCommandOutput> {
217+
const copyOut = await this.copy(name, dest);
218+
const params: DeleteObjectCommandInput = { Bucket: this.bucket, Key: name };
219+
await this.client.send(new DeleteObjectCommand(params));
220+
return copyOut;
221+
}
222+
223+
buildCompletedFile(file: S3File): S3File {
224+
const completed = { ...file };
225+
completed.lock = async lockFn => {
226+
completed.lockedBy = lockFn;
227+
return Promise.resolve(completed.lockedBy);
228+
};
229+
completed.delete = () => this.delete(file.name);
230+
completed.copy = async (dest: string) => this.copy(file.name, dest);
231+
completed.move = async (dest: string) => this.move(file.name, dest);
232+
233+
return completed;
234+
}
235+
200236
protected _onComplete = (file: S3File): Promise<[CompleteMultipartUploadOutput, any]> => {
201237
return Promise.all([this._complete(file), this.deleteMeta(file.name)]);
202238
};

‎test/gcs-storage.spec.ts‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ describe('GCStorage', () => {
3030
let storage: GCStorage;
3131
let file: GCSFile;
3232
const uri = 'http://api.com?upload_id=123456789';
33-
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } });
33+
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } as GCSFile });
3434
const _createResponse = (): any => ({ headers: { location: uri } });
3535
const req = { headers: { origin: 'http://api.com' } } as IncomingMessage;
3636

3737
beforeEach(async () => {
3838
mockAuthRequest.mockResolvedValueOnce({ bucket: 'ok' });
39-
storage = new GCStorage({ ...(storageOptions as GCStorageOptions) });
39+
storage = new GCStorage({ ...(storageOptions as GCStorageOptions), bucket: 'test' });
4040
file = _fileResponse().data;
4141
});
4242

@@ -79,13 +79,13 @@ describe('GCStorage', () => {
7979
});
8080
});
8181

82-
describe('.get()', () => {
82+
describe('.list()', () => {
8383
it('should return all user files', async () => {
8484
const list = {
8585
data: { items: [{ name: metafile }] }
8686
};
8787
mockAuthRequest.mockResolvedValue(list);
88-
const { items } = await storage.get(testfile.userId);
88+
const { items } = await storage.list(testfile.userId);
8989
expect(items).toEqual(expect.any(Array));
9090
expect(items).toHaveLength(1);
9191
expect(items[0]).toMatchObject({ name: filename });
@@ -144,6 +144,32 @@ describe('GCStorage', () => {
144144
expect(deleted.status).toBe('deleted');
145145
});
146146
});
147+
148+
describe('.copy()', () => {
149+
it('relative', async () => {
150+
mockAuthRequest.mockResolvedValue({ data: { done: true } });
151+
await storage.copy(filename, 'files/новое имя.txt');
152+
expect(mockAuthRequest).toHaveBeenCalledWith({
153+
body: '',
154+
headers: { 'Content-Type': 'application/json' },
155+
method: 'POST',
156+
url:
157+
'https://storage.googleapis.com/storage/v1/b/test/o/userId/testfile.mp4/rewriteTo/b/test/o/userId/' +
158+
'files/%D0%BD%D0%BE%D0%B2%D0%BE%D0%B5%20%D0%B8%D0%BC%D1%8F.txt'
159+
});
160+
});
161+
162+
it('absolute', async () => {
163+
mockAuthRequest.mockResolvedValue({ data: { done: true } });
164+
await storage.copy(filename, '/new/name.txt');
165+
expect(mockAuthRequest).toHaveBeenCalledWith({
166+
body: '',
167+
headers: { 'Content-Type': 'application/json' },
168+
method: 'POST',
169+
url: 'https://storage.googleapis.com/storage/v1/b/test/o/userId/testfile.mp4/rewriteTo/b/new/o/name.txt'
170+
});
171+
});
172+
});
147173
});
148174

149175
describe('Range utils', () => {

0 commit comments

Comments
 (0)