Skip to content

Commit d648919

Browse files
author
transcai
committed
support redis queue
1 parent 83398b5 commit d648919

File tree

10 files changed

+266
-8
lines changed

10 files changed

+266
-8
lines changed

‎README.md‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@
4141

4242
- SALT：接口的签名 Key，注意保密。开启后请求参数需要增加 sign 参数，sign 根据 SALT 生成（sign 参数具体生成规则见下文）
4343

44+
- REDIS_QUEUE：是否开启 redis 队列存储方式，默认为不开启，使用内存队列存储，建议业务上线后开启 redis 队列存储实现多机部署
45+
46+
如开启则需要填写 redis 服务地址和密码，不开启则无需填写
47+
48+
- REDIS：redis 服务连接地址
49+
50+
- REDIS_PWD：redis 服务密码，如果没有密码可为空
51+
4452
### 3. 启动服务
4553

4654
配置项输入完成后会自动启动服务，控制台如输出 gs-server-demo@0.0.0 start 则表示启动成功
@@ -82,7 +90,7 @@ docker run -d -p3000:3000 cgserver
8290
使用环境变量输入参数（如已生成 config.json, 不需要再设置环境变量）：
8391

8492
```bash
85-
docker run -d -p3000:3000 -e SECRET_KEY=xxx -e SECRET_ID=yyy -e SALT=zzz cgserver
93+
docker run -d -p3000:3000 -e SECRET_KEY=xxx -e SECRET_ID=yyy cgserver
8694
```
8795

8896
支持的环境变量如下：
@@ -97,6 +105,14 @@ docker run -d -p3000:3000 -e SECRET_KEY=xxx -e SECRET_ID=yyy -e SALT=zzz cgserve
97105

98106
- SALT：接口的签名 Key，注意保密。开启后请求参数需要增加 sign 参数，sign 根据 SALT 生成（sign 参数具体生成规则见下文）
99107

108+
- REDIS_QUEUE：是否开启 redis 队列存储方式，默认为不开启，使用内存队列存储，建议业务上线后开启 redis 队列存储实现多机部署
109+
110+
如开启则需要填写 redis 服务地址和密码，不开启则无需填写
111+
112+
- REDIS：redis 服务连接地址
113+
114+
- REDIS_PWD：redis 服务密码，如果没有密码可为空
115+
100116
## 接口请求类型
101117

102118
- 请求方法：HTTP POST

‎cloud_rendering_lib/base_queue.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ class BaseQueue {
4242
}
4343

4444
checkQueue() {
45-
LOG.warn('BaseQueue');
4645
}
4746

4847
addCallback(key, cb) {

‎cloud_rendering_lib/com.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ Config.registerModule(__filename, {
6363
sign: {
6464
type: 'string',
6565
description: '是否开启请求校验参数（Y/N），不填默认不开启',
66+
pattern: /^[YN]?$/,
6667
required: true,
6768
default: 'N'
6869
},

‎cloud_rendering_lib/config.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const CONF_KEYS = {
55
SECRET_ID: 'secret_id', // 腾讯云 SecretId
66
SECRET_KEY: 'secret_key', // 腾讯云 SecretKey
77
API_SIGN: 'api_sign', // 是否开启 sign 校验参数
8+
REDIS_QUEUE: 'redis_queue', // 是否开启 redis 队列存储方式
89
FILE_PATH: './config.json', // 配置文件名
910
};
1011

@@ -51,7 +52,6 @@ class AppConfig {
5152
}
5253

5354
async runInstall() {
54-
console.info('正在进行初始化配置······');
5555
prompt.start();
5656
prompt.message = '';
5757
for (const k in this.modules) {

‎cloud_rendering_lib/redis.js‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
const proc = require('process');
2+
const prompt = require('prompt');
3+
const Redis = require('redis');
4+
const { Config, DefaultKeys } = require('./config');
5+
const { validString } = require('./com');
6+
const LOG = require('./log');
7+
8+
const kRedisQueueEnv = 'REDIS_QUEUE';
9+
const kRedis = 'redis';
10+
const kRedisEnv = 'REDIS';
11+
const kRedisPwd = 'redis_pwd';
12+
const kRedisPwdEnv = 'REDIS_PWD';
13+
14+
Config.registerModule(__filename, {
15+
loadEnv: _ => {
16+
const conf = {};
17+
if (validString(proc.env[kRedisQueueEnv])) {
18+
conf[DefaultKeys.REDIS_QUEUE] = proc.env[kRedisQueueEnv];
19+
}
20+
if (validString(proc.env[kRedisEnv])) {
21+
conf[kRedis] = proc.env[kRedisEnv];
22+
}
23+
if (validString(proc.env[kRedisPwdEnv])) {
24+
conf[kRedisPwd] = proc.env[kRedisPwdEnv];
25+
}
26+
return conf;
27+
},
28+
install: async _ => {
29+
const schema = {
30+
properties: {
31+
redis_queue: {
32+
type: 'string',
33+
description: '是否开启 redis 队列存储方式（Y/N），不填默认不开启，使用内存队列存储',
34+
pattern: /^[YN]?$/,
35+
required: true,
36+
default: 'N'
37+
},
38+
}
39+
};
40+
const ret = await prompt.get(schema);
41+
Config.set(DefaultKeys.REDIS_QUEUE, ret.redis_queue);
42+
if (Config.configs[DefaultKeys.REDIS_QUEUE] == 'Y') {
43+
const schema = {
44+
properties: {
45+
redis: {
46+
description: '请输入 redis 服务连接地址',
47+
required: true,
48+
},
49+
redisPwd: {
50+
description: '请输入 redis 服务密码',
51+
required: false
52+
},
53+
}
54+
};
55+
const ret = await prompt.get(schema);
56+
Config.set(kRedis, ret.redis);
57+
Config.set(kRedisPwd, ret.redisPwd);
58+
}
59+
}
60+
});
61+
62+
class RedisConnection {
63+
constructor() {
64+
this.client = Redis.createClient(Config.get(kRedis),
65+
validString(Config.get(kRedisPwd))
66+
? { auth_pass: Config.get(kRedisPwd) } : null);
67+
68+
const client = this.client;
69+
70+
this.client.on('ready', _ => {
71+
LOG.info('redis client ready');
72+
});
73+
74+
this.client.on('connect', _ => {
75+
LOG.info('redis is now connected');
76+
});
77+
78+
this.client.on('reconnecting', (info) => {
79+
LOG.info('redis reconnecting', info);
80+
});
81+
82+
this.client.on('end', _ => {
83+
LOG.info('redis is closed');
84+
});
85+
86+
this.client.on('warning', (...args) => {
87+
LOG.warn('redis client warning', args);
88+
});
89+
90+
const onError = (err) => {
91+
LOG.error('redis raise error', err, this.client.options);
92+
};
93+
94+
this.client.on('error', onError.bind(this.client));
95+
}
96+
97+
set(key, value, expire) {
98+
return new Promise((resolve, reject) => {
99+
this.client.set(key, value, (err, doc) => {
100+
if (err) {
101+
reject(err);
102+
return;
103+
}
104+
if (typeof (expire) == 'number' && expire != -1) {
105+
this.client.expire(key, expire);
106+
}
107+
resolve(doc);
108+
});
109+
});
110+
}
111+
112+
get(key) {
113+
return new Promise((resolve, reject) =>
114+
this.client.get(key, (err, ret) => err ? reject(err) : resolve(ret)));
115+
}
116+
117+
mget(key, keys) {
118+
return new Promise((resolve, reject) =>
119+
this.client.mget(key, keys, (err, ret) => err ? reject(err) : resolve(ret)));
120+
}
121+
122+
del(key) {
123+
return new Promise((resolve, reject) =>
124+
this.client.del(key, (err, ret) => err ? reject(err) : resolve(ret)));
125+
}
126+
127+
zadd(key, score, item) {
128+
return new Promise((resolve, reject) =>
129+
this.client.zadd(key, score, item, (err, ret) => err ? reject(err) : resolve(ret)));
130+
}
131+
132+
zrank(key, item) {
133+
return new Promise((resolve, reject) =>
134+
this.client.zrank(key, item, (err, ret) => err ? reject(err) : resolve(ret)));
135+
}
136+
137+
zrem(key, item) {
138+
return new Promise((resolve, reject) =>
139+
this.client.zrem(key, item, (err, ret) => err ? reject(err) : resolve(ret)));
140+
}
141+
142+
zcard(key) {
143+
return new Promise((resolve, reject) =>
144+
this.client.zcard(key, (err, ret) => err ? reject(err) : resolve(ret)));
145+
}
146+
147+
zrange(key, start, size) {
148+
return new Promise((resolve, reject) =>
149+
this.client.zrange(key, start, size, (err, ret) => err ? reject(err) : resolve(ret)));
150+
}
151+
152+
hset(key, field, value) {
153+
return new Promise((resolve, reject) =>
154+
this.client.hset(key, field, value, (err, ret) => err ? reject(err) : resolve(ret)));
155+
}
156+
157+
hget(key, field) {
158+
return new Promise((resolve, reject) =>
159+
this.client.hget(key, field, (err, ret) => err ? reject(err) : resolve(ret)));
160+
}
161+
162+
hdel(key, field) {
163+
return new Promise((resolve, reject) =>
164+
this.client.hdel(key, field, (err, ret) => err ? reject(err) : resolve(ret)));
165+
}
166+
};
167+
168+
let redisConnection = null;
169+
170+
module.exports = {
171+
createRedisConnection: _ => {
172+
if (!redisConnection) {
173+
redisConnection = new RedisConnection();
174+
}
175+
},
176+
getRedisConnection: _ => {
177+
return redisConnection;
178+
}
179+
};

‎cloud_rendering_lib/redis_queue.js‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
const BaseQueue = require('./base_queue');
2+
const LOG = require('./log');
3+
const { getRedisConnection } = require("./redis");
4+
5+
class RedisQueue extends BaseQueue {
6+
constructor(queueName, checkInterval) {
7+
super(checkInterval);
8+
this.queueName = queueName;
9+
}
10+
11+
async checkQueue() {
12+
if (!getRedisConnection()) {
13+
this.nextCheck();
14+
return;
15+
}
16+
try {
17+
const items = await getRedisConnection().zrange(this.queueName, 0, 1);
18+
if (items.length > 0) {
19+
const item = items[0];
20+
if (await this.canDequeue(item)) {
21+
await this.dequeue(item);
22+
}
23+
}
24+
} catch (e) {
25+
LOG.warn('raise except:', e);
26+
}
27+
this.nextCheck();
28+
}
29+
30+
async enqueue(key, checkQueueDoneCallback) {
31+
this.addCallback(key, checkQueueDoneCallback);
32+
return await getRedisConnection().zadd(this.queueName, Date.now(), key);
33+
}
34+
35+
async dequeue(key) {
36+
this.removeCallback(key);
37+
return await getRedisConnection().zrem(this.queueName, key);
38+
}
39+
40+
async indexOf(key) {
41+
return await getRedisConnection().zrank(this.queueName, key);
42+
}
43+
44+
async count() {
45+
return await getRedisConnection().zcard(this.queueName);
46+
}
47+
}
48+
49+
module.exports = RedisQueue;

‎entry.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
const { Config } = require('./cloud_rendering_lib/config');
2-
require('./routes/gs');
3-
42
Config.reloadConfig();
3+
4+
require('./routes/gs');

‎install.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
const process = require('process');
1+
require('./cloud_rendering_lib/com');
2+
require('./cloud_rendering_lib/sign');
3+
require('./cloud_rendering_lib/redis');
24

5+
const process = require('process');
36
const { Config } = require('./cloud_rendering_lib/config');
4-
require('./entry');
7+
Config.reloadConfig();
58

69
(async _ => {
710
await Config.runInstall();

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"http-errors": "~1.6.3",
1515
"mongoose": "^6.1.6",
1616
"morgan": "~1.9.1",
17+
"redis": "^3.1.2",
1718
"prompt": "^1.2.1",
1819
"tencentcloud-sdk-nodejs": "^4.0.274",
1920
"uuid": "^8.3.2"

‎routes/gs.js‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,27 @@ const {
1414
simpleRespone,
1515
onMissParams } = require('../cloud_rendering_lib/com');
1616
const { verifySign } = require('../cloud_rendering_lib/sign');
17+
const { createRedisConnection } = require('../cloud_rendering_lib/redis');
1718
const { Config, DefaultKeys } = require('../cloud_rendering_lib/config');
1819
const RequestConstraint = require('../cloud_rendering_lib/constraint');
20+
const BaseQueue = require('../cloud_rendering_lib/base_queue');
1921
const MemQueue = require('../cloud_rendering_lib/mem_queue');
22+
const RedisQueue = require('../cloud_rendering_lib/redis_queue');
2023
const LOG = require('../cloud_rendering_lib/log');
2124

2225
let apiParamsSchema = {};
2326
const waitQueue = {};
2427
const enqueueTimeout = 30000; // ms
2528
const queueCheckInterval = 1000; // ms
2629
const noIdleMsg = 'ResourceNotFound.NoIdle';
27-
const queue = new MemQueue(queueCheckInterval);
30+
31+
queue = new BaseQueue(queueCheckInterval)
32+
if (Config.configs[DefaultKeys.REDIS_QUEUE] == 'Y') {
33+
createRedisConnection();
34+
queue = new RedisQueue("WaitQueue", queueCheckInterval);
35+
} else {
36+
queue = new MemQueue(queueCheckInterval);
37+
}
2838

2939
if (Config.configs[DefaultKeys.API_SIGN] == 'Y') {
3040
apiParamsSchema = {

0 commit comments

Comments
 (0)