Skip to content

Commit af9350e

Browse files
e-khalilovirdkwmnsbDaniilSmirnove.khalilovAlk2017
authored
Remove all DB connections from device side (#368)
* hotfix imei * Decrease apt install list in dockerfile (#365) * remove gm and jdk11 * remove libs * Remove console-feed & react dependencies [backend only] (#366) * remove dep console-feed * minor fix * linter fix --------- Co-authored-by: e.khalilov <e.khalilov@vkteam.ru> * fix default quotas hierarchy QA-19255 (#367) * -fix quotas -fix lock -fix test -fix lint * -fix test --------- Co-authored-by: a.chistov <a.chistov@vk.team> * dev units without db conn * fix types * minor fix * minor fix * minor fix * remove useless sockets --------- Co-authored-by: Maksim Alzhanov <m.alzhanov@vk.team> Co-authored-by: Maxim <me@alzhanov.ru> Co-authored-by: Daniil <8039921+DaniilSmirnov@users.noreply.github.com> Co-authored-by: e.khalilov <e.khalilov@vkteam.ru> Co-authored-by: Alexey Chistov <33050834+Alk2017@users.noreply.github.com> Co-authored-by: a.chistov <a.chistov@vk.team>
1 parent 0fa7a73 commit af9350e

52 files changed

Lines changed: 2354 additions & 1529 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Dockerfile

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,7 @@ WORKDIR /app
55

66
RUN apt-get update && apt-get install -y \
77
python3 \
8-
build-essential \
98
git \
10-
graphicsmagick \
11-
openjdk-11-jdk \
12-
yasm \
13-
libzmq3-dev \
14-
libprotobuf-dev \
159
&& rm -rf /var/lib/apt/lists/*
1610

1711
COPY . .

docker-compose-macos.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ services:
108108
container_name: devicehub-processor
109109
env_file:
110110
- scripts/variables.env
111-
command: stf processor --name processor --connect-app-dealer tcp://devicehub-triproxy-app:7160 --connect-dev-dealer tcp://devicehub-triproxy-dev:7260 --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270
111+
command: stf processor --name processor --connect-app-dealer tcp://devicehub-triproxy-app:7160 --connect-dev-dealer tcp://devicehub-triproxy-dev:7260
112112
depends_on:
113113
devicehub-migrate:
114114
condition: service_completed_successfully
@@ -121,7 +121,7 @@ services:
121121
container_name: devicehub-reaper
122122
env_file:
123123
- scripts/variables.env
124-
command: stf reaper --name reaper001 --connect-push tcp://devicehub-triproxy-dev:7270 --connect-sub tcp://devicehub-triproxy-app:7150 --heartbeat-timeout 30000
124+
command: stf reaper --name reaper001 --connect-push tcp://devicehub-triproxy-dev:7270 --connect-sub tcp://devicehub-triproxy-dev:7250 --heartbeat-timeout 30000
125125
depends_on:
126126
devicehub-migrate:
127127
condition: service_completed_successfully
@@ -233,7 +233,7 @@ services:
233233
container_name: devicehub-api-groups-engine
234234
env_file:
235235
- scripts/variables.env
236-
command: stf groups-engine --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270
236+
command: stf groups-engine --connect-push tcp://devicehub-triproxy-app:7170 --connect-push-dev tcp://devicehub-triproxy-dev:7270
237237
depends_on:
238238
devicehub-migrate:
239239
condition: service_completed_successfully
@@ -263,4 +263,4 @@ services:
263263
- certs:/certs:rw
264264
volumes:
265265
devicehub-db-volume:
266-
certs:
266+
certs:

docker-compose-prod.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ services:
124124
container_name: devicehub-processor
125125
env_file:
126126
- scripts/variables.env
127-
command: stf processor --name processor --connect-app-dealer tcp://devicehub-triproxy-app:7160 --connect-dev-dealer tcp://devicehub-triproxy-dev:7260 --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270
127+
command: stf processor --name processor --connect-app-dealer tcp://devicehub-triproxy-app:7160 --connect-dev-dealer tcp://devicehub-triproxy-dev:7260
128128
depends_on:
129129
devicehub-migrate:
130130
condition: service_completed_successfully
@@ -137,7 +137,7 @@ services:
137137
container_name: devicehub-reaper
138138
env_file:
139139
- scripts/variables.env
140-
command: stf reaper --name reaper001 --connect-push tcp://devicehub-triproxy-dev:7270 --connect-sub tcp://devicehub-triproxy-app:7150 --heartbeat-timeout 30000
140+
command: stf reaper --name reaper001 --connect-push tcp://devicehub-triproxy-dev:7270 --connect-sub tcp://devicehub-triproxy-dev:7250 --heartbeat-timeout 30000
141141
depends_on:
142142
devicehub-migrate:
143143
condition: service_completed_successfully
@@ -249,7 +249,7 @@ services:
249249
container_name: devicehub-api-groups-engine
250250
env_file:
251251
- scripts/variables.env
252-
command: stf groups-engine --connect-sub tcp://devicehub-triproxy-app:7150 --connect-push tcp://devicehub-triproxy-app:7170 --connect-sub-dev tcp://devicehub-triproxy-dev:7250 --connect-push-dev tcp://devicehub-triproxy-dev:7270
252+
command: stf groups-engine --connect-push tcp://devicehub-triproxy-app:7170 --connect-push-dev tcp://devicehub-triproxy-dev:7270
253253
depends_on:
254254
devicehub-migrate:
255255
condition: service_completed_successfully

lib/cli/doctor/index.js

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,6 @@ export const handler = function() {
111111
return checker.version(pkg.externalDependencies.monogdb)(pkg.dependencies.mongodb)
112112
})
113113
}
114-
function checkGraphicsMagick() {
115-
return check('GraphicsMagick', function(checker) {
116-
return checker.call('gm', ['-version'])
117-
.then(checker.extract('version', /GraphicsMagick ([^\s]+)/))
118-
.then(checker.version(pkg.externalDependencies.gm))
119-
})
120-
}
121114
function checkZeroMQ() {
122115
return check('ZeroMQ', function(checker) {
123116
return checker.version(pkg.externalDependencies.zeromq)(zmq.version)
@@ -143,7 +136,6 @@ export const handler = function() {
143136
checkOSRelease(),
144137
checkNodeVersion(),
145138
checkLocalMongoDBVersion(),
146-
checkGraphicsMagick(),
147139
checkZeroMQ(),
148140
checkProtoBuf(),
149141
checkADB()

lib/cli/groups-engine/index.js

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,12 @@ export const builder = function(yargs) {
1111
array: true,
1212
demand: true
1313
})
14-
.option('connect-sub', {
15-
alias: 'u',
16-
describe: 'App-side ZeroMQ PUB endpoint to connect to.',
17-
array: true,
18-
demand: true
19-
})
2014
.option('connect-push-dev', {
2115
alias: 'pd',
2216
describe: 'Device-side ZeroMQ PULL endpoint to connect to.',
2317
array: true,
2418
demand: true
2519
})
26-
.option('connect-sub-dev', {
27-
alias: 'sd',
28-
describe: 'Device-side ZeroMQ PUB endpoint to connect to.',
29-
array: true,
30-
demand: true
31-
})
3220
.epilog('Each option can be be overwritten with an environment variable ' +
3321
'by converting the option to uppercase, replacing dashes with ' +
3422
'underscores and prefixing it with `STF_GROUPS_ENGINE_` .)')
@@ -37,9 +25,7 @@ export const handler = function(argv) {
3725
return groupsEngine({
3826
endpoints: {
3927
push: argv.connectPush,
40-
sub: argv.connectSub,
4128
pushdev: argv.connectPushDev,
42-
subdev: argv.connectSubDev
4329
}
4430
})
4531
}

lib/cli/local/index.js

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -338,16 +338,12 @@ export const handler = function(argv) {
338338
'processor', 'proc001',
339339
'--connect-app-dealer', argv.bindAppDealer,
340340
'--connect-dev-dealer', argv.bindDevDealer,
341-
'--connect-push', argv.bindAppPull,
342-
'--connect-push-dev', argv.bindDevPull,
343-
'--connect-sub', argv.bindAppPub,
344-
'--connect-sub-dev', argv.bindDevPub,
345341
'--public-ip', argv.publicIp
346342
],
347343
[ // reaper one
348344
'reaper', 'reaper001',
349345
'--connect-push', argv.bindDevPull,
350-
'--connect-sub', argv.bindAppPub
346+
'--connect-sub', argv.bindDevPub
351347
],
352348
[ // provider
353349
'provider',
@@ -412,9 +408,7 @@ export const handler = function(argv) {
412408
[ // groups engine
413409
'groups-engine',
414410
'--connect-push', argv.bindAppPull,
415-
'--connect-sub', argv.bindAppPub,
416411
'--connect-push-dev', argv.bindDevPull,
417-
'--connect-sub-dev', argv.bindDevPub
418412
],
419413
[ // websocket
420414
'websocket',

lib/cli/processor/index.js

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,30 +18,6 @@ export const builder = function(yargs) {
1818
array: true,
1919
demand: true
2020
})
21-
.option('connect-push', {
22-
alias: 'c',
23-
describe: 'App-side ZeroMQ PULL endpoint to connect to.',
24-
array: true,
25-
demand: true
26-
})
27-
.option('connect-push-dev', {
28-
alias: 'pd',
29-
describe: 'Device-side ZeroMQ PULL endpoint to connect to.',
30-
array: true,
31-
demand: true
32-
})
33-
.option('connect-sub', {
34-
alias: 'u',
35-
describe: 'App-side ZeroMQ PUB endpoint to connect to.',
36-
array: true,
37-
demand: true
38-
})
39-
.option('connect-sub-dev', {
40-
alias: 'sd',
41-
describe: 'Device-side ZeroMQ PUB endpoint to connect to.',
42-
array: true,
43-
demand: true
44-
})
4521
.option('name', {
4622
describe: 'An easily identifiable name for log output.',
4723
type: 'string',
@@ -64,10 +40,6 @@ export const handler = function(argv) {
6440
endpoints: {
6541
appDealer: argv.connectAppDealer,
6642
devDealer: argv.connectDevDealer,
67-
push: argv.connectPush,
68-
pushdev: argv.connectPushDev,
69-
sub: argv.connectSub,
70-
subdev: argv.connectSubDev
7143
},
7244
publicIp: argv.publicIp
7345
})

lib/db/index.ts

Lines changed: 62 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -117,21 +117,18 @@ export default class DbClient {
117117
pushdev,
118118
channelRouter,
119119
}: {
120-
sub: SocketWrapper | string[];
121-
subdev: SocketWrapper | string[];
120+
sub?: SocketWrapper | string[];
121+
subdev?: SocketWrapper | string[];
122122
push: SocketWrapper | string[];
123123
pushdev: SocketWrapper | string[];
124124
channelRouter?: EventEmitter;
125125
},
126126
_log: ReturnType<typeof logger.createLogger> | undefined = log
127-
): Promise<{
128-
sub: SocketWrapper;
129-
subdev: SocketWrapper;
130-
push: SocketWrapper;
131-
pushdev: SocketWrapper;
132-
channelRouter: EventEmitter;
133-
}> {
134-
if (Array.isArray(sub)) {
127+
) {
128+
let finalSub: SocketWrapper | undefined
129+
let finalSubdev: SocketWrapper | undefined
130+
131+
if (sub && Array.isArray(sub)) {
135132
const _sub = zmqutil.socket('sub')
136133
await Promise.all(
137134
sub.map(async(endpoint) => {
@@ -153,10 +150,13 @@ export default class DbClient {
153150
}
154151
})
155152
)
156-
sub = _sub
153+
finalSub = _sub
154+
}
155+
else if (sub) {
156+
finalSub = sub as SocketWrapper
157157
}
158158

159-
if (Array.isArray(subdev)) {
159+
if (subdev && Array.isArray(subdev)) {
160160
const _subdev = zmqutil.socket('sub')
161161
await Promise.all(
162162
subdev.map(async(endpoint) => {
@@ -178,12 +178,15 @@ export default class DbClient {
178178
}
179179
})
180180
)
181-
subdev = _subdev
181+
finalSubdev = _subdev
182+
}
183+
else if (subdev) {
184+
finalSubdev = subdev as SocketWrapper
182185
}
183186

184187
if (Array.isArray(push)) {
185188
const _push = zmqutil.socket('push')
186-
Promise.all(
189+
await Promise.all(
187190
push.map(async(endpoint) => {
188191
try {
189192
srv.attempt(
@@ -205,7 +208,7 @@ export default class DbClient {
205208

206209
if (Array.isArray(pushdev)) {
207210
const _pushdev = zmqutil.socket('push')
208-
Promise.all(
211+
await Promise.all(
209212
pushdev.map(async(endpoint) => {
210213
try {
211214
srv.attempt(
@@ -229,23 +232,48 @@ export default class DbClient {
229232
}
230233

231234
if (!channelRouter) {
232-
channelRouter = new EventEmitter();
233-
[wireutil.global].forEach((channel) => {
234-
_log.info('Subscribing to permanent channel "%s"', channel)
235-
sub.subscribe(channel)
236-
subdev.subscribe(channel)
237-
})
235+
channelRouter = new EventEmitter()
238236

239-
sub.on('message', (channel, data) => {
240-
channelRouter?.emit(channel.toString(), channel, data)
241-
})
237+
if (finalSub || finalSubdev) {
238+
;[wireutil.global].forEach((channel) => {
239+
_log.info('Subscribing to permanent channel "%s"', channel)
240+
if (finalSub) {
241+
finalSub.subscribe(channel)
242+
}
243+
if (finalSubdev) {
244+
finalSubdev.subscribe(channel)
245+
}
246+
})
247+
248+
if (finalSub) {
249+
finalSub.on('message', (channel, data) => {
250+
channelRouter?.emit(channel.toString(), channel, data)
251+
})
252+
}
242253

243-
subdev.on('message', (channel, data) => {
244-
channelRouter?.emit(channel.toString(), channel, data)
245-
})
254+
if (finalSubdev) {
255+
finalSubdev.on('message', (channel, data) => {
256+
channelRouter?.emit(channel.toString(), channel, data)
257+
})
258+
}
259+
}
246260
}
247261

248-
return {sub, subdev, push, pushdev, channelRouter}
262+
const result: {
263+
sub?: SocketWrapper;
264+
subdev?: SocketWrapper;
265+
push: SocketWrapper;
266+
pushdev: SocketWrapper;
267+
channelRouter: EventEmitter;
268+
} = {
269+
push,
270+
pushdev,
271+
channelRouter,
272+
... !!finalSub && {sub: finalSub},
273+
... !!finalSubdev && {subdev: finalSubdev}
274+
}
275+
276+
return result
249277
}
250278

251279
// Verifies that we can form a connection. Useful if it's necessary to make
@@ -254,11 +282,12 @@ export default class DbClient {
254282
// an issue with the processor unit, as it started processing messages before
255283
// it was actually truly able to save anything to the database. This lead to
256284
// lost messages in certain situations.
257-
static ensureConnectivity = async <T extends Function>(fn: T) => {
258-
await DbClient.connect()
259-
log.info("Db is up")
260-
return fn
261-
}
285+
static ensureConnectivity = <T extends (...args: any[]) => any>(fn: T) =>
286+
async(...args: Parameters<T>): Promise<ReturnType<T>> => {
287+
await DbClient.connect();
288+
log.info("Db is up");
289+
return fn(...args);
290+
}
262291

263292
// Sets up the database
264293
static setup = () => DbClient.connect().then((conn) => _setup(conn))

0 commit comments

Comments
 (0)