Skip to content

Commit a27f656

Browse files
zeeshanakram3zeeshanakram3
committed
Merge remote-tracking branch 'upstream/master' into origo_argus_colossus_operational_status_feat
2 parents 21bc807 + a2fae87 commit a27f656

47 files changed

Lines changed: 68338 additions & 237 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dockerignore‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ target/
44
.vscode/
55
query-node/**/dist
66
query-node/lib
7-
tests/
8-
!tests/network-tests/proposal-parameters.json
97
devops/
108
!devops/eslint-config
11-
!devops/prettier-config
9+
!devops/prettier-config

‎.github/workflows/joystream-apps-docker.yml‎

Lines changed: 86 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,102 @@
1-
name: Build joystream/apps and publish to Docker Hub
1+
name: Publish Colossus/Argus/QueryNode Docker images
22

33
on:
44
workflow_dispatch:
5-
inputs:
6-
tag_suffix:
7-
description: 'Tag suffix'
8-
required: true
5+
pull_request:
6+
97
jobs:
10-
docker:
8+
build_and_publish:
9+
name: Build and Publish
1110
runs-on: ubuntu-latest
1211
steps:
1312
- name: Checkout
1413
uses: actions/checkout@v3
15-
- name: Extract branch name
14+
15+
- uses: actions/setup-node@v3
16+
with:
17+
node-version: '18.x'
18+
19+
- name: Extract Package Versions
20+
id: extract_versions
1621
shell: bash
17-
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
18-
id: extract_branch
22+
run: |
23+
echo "colossus_version=$(cat storage-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
24+
echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
25+
echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
26+
27+
- name: Set up Docker Buildx
28+
uses: docker/setup-buildx-action@v2
1929
- name: Login to DockerHub
30+
if: github.event_name == 'workflow_dispatch'
2031
uses: docker/login-action@v2
2132
with:
2233
username: ${{ secrets.DOCKERHUB_USERNAME }}
2334
password: ${{ secrets.DOCKERHUB_PASSWORD }}
24-
- name: Build and push
25-
id: docker_build
35+
36+
- name: Build joystream-node
37+
run: RUNTIME_PROFILE=TESTING ./build-node-docker.sh
38+
39+
# docker/build-push-action doc:
40+
# Be careful because any file mutation in the steps that precede the
41+
# build step will be ignored, including processing of the .dockerignore file
42+
# since the context is based on the Git reference. However, you can use
43+
# the Path context using the context input alongside the actions/checkout action
44+
# to remove this restriction.
45+
- name: Build storage-node
2646
uses: docker/build-push-action@v3
2747
with:
28-
file: apps.Dockerfile
29-
push: true
30-
tags: joystream/apps:${{ steps.extract_branch.outputs.branch }}-${{ github.event.inputs.tag_suffix }}
48+
# Do not use local dir context to ensure we can build from a commit directly
49+
# context: .
50+
file: colossus.Dockerfile
51+
push: false
52+
load: true
53+
tags: joystream/storage-node:latest
54+
- name: Build distributor-node
55+
uses: docker/build-push-action@v3
56+
with:
57+
# Do not use local dir context to ensure we can build from a commit directly
58+
# context: .
59+
file: distributor-node.Dockerfile
60+
push: false
61+
load: true
62+
tags: joystream/distributor-node:latest
63+
- name: Build query-node
64+
uses: docker/build-push-action@v3
65+
with:
66+
# Do not use local dir context to ensure we can build from a commit directly
67+
# context: .
68+
file: query-node.Dockerfile
69+
push: false
70+
load: true
71+
tags: joystream/query-node:latest
72+
73+
- name: Test with latest images
74+
run: |
75+
yarn build:packages
76+
cp docker-compose-no-bind-volumes.yml docker-compose.yml
77+
./tests/network-tests/run-test-node-docker.sh
78+
sleep 15
79+
docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
80+
sleep 30
81+
./tests/network-tests/start-storage.sh
82+
export REUSE_KEYS=true
83+
export SKIP_STORAGE_AND_DISTRIBUTION=true
84+
IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
85+
86+
- name: Push new versions
87+
if: github.event_name == 'workflow_dispatch'
88+
run: |
89+
docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
90+
docker image tag joystream/distributor-node:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
91+
docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
92+
docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
93+
docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
94+
docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
95+
96+
- name: Push latest tag
97+
if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
98+
run: |
99+
docker push joystream/storage-node:latest
100+
docker push joystream/distributor:latest
101+
docker push joystream/query-node:latest
102+

‎.github/workflows/joystream-node-docker.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636
export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh`
3737
echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}"
3838
39-
# docker manifest inspect command requires authentication
39+
# docker manifest inspect command requires authentication
4040
- name: Login to DockerHub
4141
uses: docker/login-action@v2
4242
with:

‎apps.Dockerfile‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎build-packages.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ yarn --frozen-lockfile
66
yarn workspace @joystream/types build
77
yarn workspace @joystream/metadata-protobuf build
88
yarn workspace @joystream/js build
9+
yarn workspace @joystream/opentelemetry build
910
yarn workspace query-node-root build
1011
yarn workspace @joystream/cli build
1112
yarn workspace storage-node build

‎cli/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ $ npm install -g @joystream/cli
2626
$ joystream-cli COMMAND
2727
running command...
2828
$ joystream-cli (-v|--version|version)
29-
@joystream/cli/1.2.0 darwin-x64 node-v14.18.0
29+
@joystream/cli/1.2.0 darwin-x64 node-v14.16.1
3030
$ joystream-cli --help [COMMAND]
3131
USAGE
3232
$ joystream-cli COMMAND

‎colossus.Dockerfile‎

Lines changed: 31 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,44 @@
1-
FROM --platform=linux/x86-64 node:14 as builder
1+
FROM --platform=linux/x86-64 node:18 as builder
22

33
WORKDIR /joystream
4-
COPY . /joystream
4+
5+
COPY ./chain-metadata.json chain-metadata.json
6+
COPY ./yarn.lock yarn.lock
7+
COPY ./types/package.json types/package.json
8+
COPY ./metadata-protobuf/package.json metadata-protobuf/package.json
9+
COPY ./devops/eslint-config/package.json ./devops/eslint-config/package.json
10+
COPY ./devops/prettier-config/package.json ./devops/prettier-config/package.json
11+
COPY ./storage-node/package.json storage-node/package.json
12+
COPY ./storage-node/client/package.json storage-node/client/package.json
13+
COPY ./opentelemetry/package.json opentelemetry/package.json
14+
COPY ./package.json package.json
515

616
RUN yarn --frozen-lockfile
717

18+
COPY ./types types
19+
COPY ./metadata-protobuf metadata-protobuf
20+
COPY ./devops/eslint-config ./devops/eslint-config
21+
COPY ./devops/prettier-config ./devops/prettier-config
22+
COPY ./storage-node storage-node
23+
COPY ./opentelemetry opentelemetry
24+
COPY ./tsconfig.json ./tsconfig.json
25+
826
RUN yarn workspace @joystream/types build
927
RUN yarn workspace @joystream/metadata-protobuf build
28+
RUN yarn workspace @joystream/opentelemetry build
1029
RUN yarn workspace storage-node build
30+
RUN yarn cache clean
31+
32+
FROM --platform=linux/x86-64 node:18 as final
33+
WORKDIR /joystream
34+
COPY --from=builder /joystream /joystream
35+
RUN yarn --frozen-lockfile --production
1136

1237
# Use these volumes to persist uploading data and to pass the keyfile.
1338
VOLUME ["/data", "/keystore", "/logs"]
1439

15-
# Required variables
16-
ENV WS_PROVIDER_ENDPOINT_URI=ws://not-set
17-
ENV COLOSSUS_PORT=3333
18-
ENV QUERY_NODE_ENDPOINT=http://not-set/graphql
19-
ENV WORKER_ID=not-set
20-
# - set external key file using the `/keystore` volume
21-
ENV ACCOUNT_KEYFILE=
22-
ENV ACCOUNT_PWD=
23-
# Optional variables
24-
ENV SYNC_INTERVAL=1
25-
ENV ELASTIC_SEARCH_ENDPOINT=
26-
# warn, error, debug, info
27-
ENV ELASTIC_LOG_LEVEL=debug
28-
# - overrides account key file
29-
ENV ACCOUNT_URI=
30-
3140
# Colossus node port
32-
EXPOSE ${COLOSSUS_PORT}
33-
34-
WORKDIR /joystream/storage-node
35-
ENTRYPOINT yarn storage-node server --queryNodeEndpoint ${QUERY_NODE_ENDPOINT} \
36-
--port ${COLOSSUS_PORT} --uploads /data \
37-
--apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} \
38-
--elasticSearchEndpoint=${ELASTIC_SEARCH_ENDPOINT} \
39-
--accountUri=${ACCOUNT_URI} \
40-
--worker ${WORKER_ID} \
41-
--logFilePath=/logs
41+
EXPOSE 3333
42+
43+
ENTRYPOINT ["yarn", "storage-node"]
44+
CMD ["server"]

‎distributor-node.Dockerfile‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,47 @@
1-
FROM --platform=linux/x86-64 node:14 as builder
1+
FROM --platform=linux/x86-64 node:18 as builder
22

33
WORKDIR /joystream
4-
COPY ./.git .git
4+
55
COPY ./chain-metadata.json chain-metadata.json
6+
COPY ./yarn.lock yarn.lock
7+
COPY ./types/package.json types/package.json
8+
COPY ./metadata-protobuf/package.json metadata-protobuf/package.json
9+
COPY ./devops/eslint-config/package.json ./devops/eslint-config/package.json
10+
COPY ./devops/prettier-config/package.json ./devops/prettier-config/package.json
11+
COPY ./storage-node/package.json storage-node/package.json
12+
COPY ./storage-node/client/package.json storage-node/client/package.json
13+
COPY ./distributor-node/package.json distributor-node/package.json
14+
COPY ./distributor-node/client/package.json distributor-node/client/package.json
15+
COPY ./opentelemetry/package.json opentelemetry/package.json
16+
COPY ./package.json package.json
17+
18+
RUN yarn --frozen-lockfile
19+
620
COPY ./types types
721
COPY ./metadata-protobuf metadata-protobuf
8-
COPY ./query-node/generated/graphql-server/generated/schema.graphql ./query-node/generated/graphql-server/generated/schema.graphql
9-
COPY ./storage-node/client ./storage-node/client
22+
COPY ./storage-node ./storage-node
1023
COPY ./devops/eslint-config ./devops/eslint-config
1124
COPY ./devops/prettier-config ./devops/prettier-config
1225
COPY ./distributor-node distributor-node
13-
COPY ./yarn.lock yarn.lock
14-
COPY ./package.json package.json
15-
16-
EXPOSE 3334
26+
COPY ./opentelemetry opentelemetry
27+
COPY ./tsconfig.json ./tsconfig.json
1728

1829
# Build & cleanup
1930
# (must be inside a single "RUN", see: https://stackoverflow.com/questions/40212836/docker-image-larger-than-its-filesystem)
2031
RUN \
21-
yarn --frozen-lockfile &&\
2232
yarn workspace @joystream/types build &&\
2333
yarn workspace @joystream/metadata-protobuf build &&\
34+
yarn workspace @joystream/storage-node-client build &&\
35+
yarn workspace @joystream/opentelemetry build &&\
2436
yarn workspace @joystream/distributor-cli build &&\
25-
find . -name "node_modules" -type d -prune &&\
26-
yarn --frozen-lockfile --production &&\
2737
yarn cache clean
2838

39+
FROM --platform=linux/x86-64 node:18 as final
40+
WORKDIR /joystream
41+
COPY --from=builder /joystream /joystream
42+
RUN yarn --frozen-lockfile --production
43+
44+
EXPOSE 3334
45+
2946
ENTRYPOINT ["yarn", "joystream-distributor"]
3047
CMD ["start"]

‎distributor-node/.prettierignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22
lib
33
local
44
/**/*.md
5-
client
5+
client
6+
src/services/networking/query-node/schema.graphql

‎distributor-node/CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
### 1.2.0
1+
### 1.2.1
22

3+
- Integrates OpenTelemetry API/SDK with Argus for exporting improved tracing logs & metrics to Elasticsearch. Adds `./start-elasticsearch-stack.sh` script to bootstrap elasticsearch services (Elasticsearch + Kibana + APM Server) with all the required configurations.
34
- Add support for Elasticsearch authentication (`logs.elastic.auth` config section)
45
- **FIX:** Objects belonging to group `0` of LRU-SP cache were never fully dropped from the state because of a bug in `StateCacheService.dropById`
56

0 commit comments

Comments
 (0)