spider/deployment/stack.yml at master · decrypto-org/spider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
version: "3.2"

networks:
  backend:
  frontend:

volumes:
  postgres-vol:
  log-vol:

services:
  db:
    image: postgres:10
    restart: always
    command: postgres -N 200
    environment:
      POSTGRES_PASSWORD: $TDSE_DB_PASSWORD
      POSTGRES_USER: $TDSE_DB_USER
      POSTGRES_DB: $TDSE_DB_NAME
    volumes:
      - postgres-vol:/var/lib/postgresql/data
    networks:
      backend:

  server:
    image: robrunne/tdse-spider:1.0.0
    restart: always
    environment:
      TDSE_DB_NAME: $TDSE_DB_NAME
      TDSE_DB_USER: $TDSE_DB_USER
      TDSE_DB_PASSWORD: $TDSE_DB_PASSWORD
      TDSE_DB_HOST: spider_db_1
      TDSE_DB_PORT: $TDSE_DB_PORT
      TDSE_TOR_HOST: spider_tor_1
      TDSE_TOR_PORT: 9000
      TDSE_LOG_LOCATION: /home/node/log
    networks:
      backend:
        aliases:
          - database
          - db
          - postgres
      frontend:
        aliases:
          - tor
          - onionnet
    volumes:
      - log-vol:/home/node/log
      - ${TDSE_INIT_URLS}:/home/node/init_urls.csv
    command: $tdse_server_start_command
    stdin_open: true
    tty: true
    depends_on:
      - "db"
      - "tor"

  tor:
    image: robrunne/tor-proxy:1.0.0
    restart: always
    networks:
      frontend:
    ports:
      - "127.0.0.1:9077:9077"
      - "127.0.0.1:9000:9000"
    cap_add:
      - NET_ADMIN
      - NET_RAW

  extractor:
    image: robrunne/tdse-uri-extractor:1.0.0
    restart: always
    environment:
      TDSE_DB_NAME: $TDSE_DB_NAME
      TDSE_DB_USER: $TDSE_DB_USER
      TDSE_DB_PASSWORD: $TDSE_DB_PASSWORD
      TDSE_DB_HOST: spider_db_1
      TDSE_DB_PORT: $TDSE_DB_PORT
      TDSE_TOR_HOST: spider_tor_1
      TDSE_TOR_PORT: 9000
      TDSE_LOG_LOCATION: /home/node/log
    networks:
      frontend:
        aliases:
          - tor
          - onionnet
      backend:
        aliases:
          - database
          - db
          - postgres
    command: bash
    depends_on:
      - "db"
    stdin_open: true
    tty: true

  preprocessor:
    image: robrunne/tdse-data-preprocessor:1.0.0
    restart: always
    environment:
      TDSE_DB_NAME: $TDSE_DB_NAME
      TDSE_DB_USER: $TDSE_DB_USER
      TDSE_DB_PASSWORD: $TDSE_DB_PASSWORD
      TDSE_DB_HOST: spider_db_1
      TDSE_DB_PORT: $TDSE_DB_PORT
      TDSE_TOR_HOST: spider_tor_1
      TDSE_TOR_PORT: 9000
      TDSE_LOG_LOCATION: /home/node/log
    networks:
      frontend:
      backend:
    command: bash
    depends_on:
      - "db"
    stdin_open: true
    tty: true

  classifier:
    image: robrunne/tdse-classifier:1.0.0
    restart: always
    environment:
      TDSE_DB_NAME: $TDSE_DB_NAME
      TDSE_DB_USER: $TDSE_DB_USER
      TDSE_DB_PASSWORD: $TDSE_DB_PASSWORD
      TDSE_DB_HOST: spider_db_1
      TDSE_DB_PORT: $TDSE_DB_PORT
      TDSE_TOR_HOST: spider_tor_1
      TDSE_TOR_PORT: 9000
      TDSE_LOG_LOCATION: /home/node/log
    networks:
      backend:
    volumes:
      - log-vol:/home/node/log
      - $TDSE_HOST_SPIDER_REPO/classifier/outputModels:/usr/src/tdse/spider/classifier/outputModels
    command: bash
    ports:
      - "65000-65100:65000-65100"
    depends_on:
      - "db"
    stdin_open: true
    tty: true