Skip to content

Commit c6e743b

Browse files
author
Nik Mohamad Aizuddin b. Nik Azmi
committed
feat(example): Add kafka-pubsub example
1 parent 294e11b commit c6e743b

10 files changed

Lines changed: 363 additions & 0 deletions

File tree

‎kafka-pubsub/Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
FROM docker.io/node:14.16.1
2+
MAINTAINER extra2000 <https://github.com/extra2000>
3+
4+
COPY ./project /srv/project
5+
WORKDIR /srv/project
6+
RUN npm install

‎kafka-pubsub/README.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Sample apps: Kafka Pub/Sub
2+
3+
4+
## Deploy Kafka pod
5+
6+
```
7+
$ podman play kube --network=sampleapps kafka-pod.yaml
8+
```
9+
10+
To test Kafka deployments:
11+
```
12+
$ podman run -it --rm --network=sampleapps docker.io/bitnami/kafka:2.6.0 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper kafka-pod:2181 --replication-factor 1 --partitions 1 --topic testingtopic
13+
$ podman run -it --rm --network=sampleapps docker.io/bitnami/kafka:2.6.0 bash -c 'echo "Hello, World" | /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server kafka-pod:9092 --topic testingtopic'
14+
```
15+
16+
For this test, press `CTRL` + `C` to exit:
17+
```
18+
$ podman run -it --rm --network=sampleapps docker.io/bitnami/kafka:2.6.0 /opt/bitnami/kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka-pod:9092 --topic testingtopic --from-beginning
19+
```
20+
21+
## Deploy producer and consumer services
22+
23+
```
24+
$ podman build -t extra2000/prodcon:latest .
25+
$ podman play kube --network=sampleapps producer-pod.yaml
26+
$ podman play kube --network=sampleapps consumer-pod.yaml
27+
```
28+
29+
See logs:
30+
```
31+
$ podman logs producer-pod-producer
32+
$ podman logs consumer-pod-consumer
33+
```
34+
35+
36+
## To clean up
37+
38+
```
39+
$ podman pod rm --force consumer-pod producer-pod kafka-pod
40+
```

‎kafka-pubsub/consumer-pod.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: consumer-pod
5+
spec:
6+
restartPolicy: Never
7+
containers:
8+
- name: consumer
9+
image: extra2000/prodcon
10+
command: ['npm', 'run', 'consumer']

‎kafka-pubsub/kafka-pod.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: kafka-pod
5+
spec:
6+
restartPolicy: Never
7+
containers:
8+
- name: zookeeper
9+
image: docker.io/bitnami/zookeeper:3.6.2
10+
ports:
11+
- containerPort: 2181
12+
protocol: tcp
13+
env:
14+
- name: ALLOW_ANONYMOUS_LOGIN
15+
value: yes
16+
- name: kafka
17+
image: docker.io/bitnami/kafka:2.6.0
18+
ports:
19+
- containerPort: 9092
20+
protocol: tcp
21+
env:
22+
- name: KAFKA_BROKER_ID
23+
value: 1
24+
- name: KAFKA_CFG_ZOOKEEPER_CONNECT
25+
value: 127.0.0.1:2181
26+
- name: ALLOW_PLAINTEXT_LISTENER
27+
value: yes
28+
- name: KAFKA_LISTENERS
29+
value: PLAINTEXT://:9092
30+
- name: KAFKA_ADVERTISED_LISTENERS
31+
value: PLAINTEXT://kafka-pod:9092 # KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092

‎kafka-pubsub/producer-pod.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: producer-pod
5+
spec:
6+
restartPolicy: Never
7+
containers:
8+
- name: producer
9+
image: extra2000/prodcon
10+
command: ['npm', 'run', 'producer']

‎kafka-pubsub/project/.gitignore‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Logs
2+
logs
3+
*.log
4+
npm-debug.log*
5+
yarn-debug.log*
6+
yarn-error.log*
7+
lerna-debug.log*
8+
9+
# Diagnostic reports (https://nodejs.org/api/report.html)
10+
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
11+
12+
# Runtime data
13+
pids
14+
*.pid
15+
*.seed
16+
*.pid.lock
17+
18+
# Directory for instrumented libs generated by jscoverage/JSCover
19+
lib-cov
20+
21+
# Coverage directory used by tools like istanbul
22+
coverage
23+
*.lcov
24+
25+
# nyc test coverage
26+
.nyc_output
27+
28+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
29+
.grunt
30+
31+
# Bower dependency directory (https://bower.io/)
32+
bower_components
33+
34+
# node-waf configuration
35+
.lock-wscript
36+
37+
# Compiled binary addons (https://nodejs.org/api/addons.html)
38+
build/Release
39+
40+
# Dependency directories
41+
node_modules/
42+
jspm_packages/
43+
44+
# TypeScript v1 declaration files
45+
typings/
46+
47+
# TypeScript cache
48+
*.tsbuildinfo
49+
50+
# Optional npm cache directory
51+
.npm
52+
53+
# Optional eslint cache
54+
.eslintcache
55+
56+
# Microbundle cache
57+
.rpt2_cache/
58+
.rts2_cache_cjs/
59+
.rts2_cache_es/
60+
.rts2_cache_umd/
61+
62+
# Optional REPL history
63+
.node_repl_history
64+
65+
# Output of 'npm pack'
66+
*.tgz
67+
68+
# Yarn Integrity file
69+
.yarn-integrity
70+
71+
# dotenv environment variables file
72+
.env
73+
.env.test
74+
75+
# parcel-bundler cache (https://parceljs.org/)
76+
.cache
77+
78+
# Next.js build output
79+
.next
80+
81+
# Nuxt.js build / generate output
82+
.nuxt
83+
dist
84+
85+
# Gatsby files
86+
.cache/
87+
# Comment in the public line in if your project uses Gatsby and *not* Next.js
88+
# https://nextjs.org/blog/next-9-1#public-directory-support
89+
# public
90+
91+
# vuepress build output
92+
.vuepress/dist
93+
94+
# Serverless directories
95+
.serverless/
96+
97+
# FuseBox cache
98+
.fusebox/
99+
100+
# DynamoDB Local files
101+
.dynamodb/
102+
103+
# TernJS port file
104+
.tern-port

‎kafka-pubsub/project/package.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"name": "kafka-pubsub",
3+
"version": "1.0.0",
4+
"description": "Kafka Pub/Sub example",
5+
"author": "extra2000",
6+
"license": "MIT",
7+
"scripts": {
8+
"tsc": "tsc",
9+
"producer": "ts-node src/producer.ts",
10+
"consumer": "ts-node src/consumer.ts"
11+
},
12+
"dependencies": {
13+
"kafka-node": "^5.0.0",
14+
"body-parser": "^1.19.0"
15+
},
16+
"devDependencies": {
17+
"typescript": "^4.0.2",
18+
"ts-node": "^9.0.0",
19+
"@types/node": "^14.11.2"
20+
}
21+
}

‎kafka-pubsub/project/src/consumer.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
const kafka = require('kafka-node');
2+
const bp = require('body-parser');
3+
4+
let kafkaHost = 'kafka-pod:9092';
5+
let kafkaTopic = 'testingtopic';
6+
7+
try {
8+
const Consumer = kafka.Consumer;
9+
const client = new kafka.KafkaClient({kafkaHost: kafkaHost});
10+
let consumer = new Consumer(
11+
client,
12+
[{ topic: kafkaTopic, partition: 0 }],
13+
{
14+
autoCommit: true,
15+
fetchMaxWaitMs: 1000,
16+
fetchMaxBytes: 1024 * 1024,
17+
encoding: 'utf8',
18+
fromOffset: false
19+
}
20+
);
21+
consumer.on('message', function(message: any) {
22+
console.log('here');
23+
console.log(
24+
'kafka-> ',
25+
message.value
26+
);
27+
})
28+
consumer.on('error', function(err: any) {
29+
console.log('error', err);
30+
});
31+
}
32+
catch(e) {
33+
console.log(e);
34+
}

‎kafka-pubsub/project/src/producer.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
const kafka = require('kafka-node');
2+
const bp = require('body-parser');
3+
4+
let kafkaHost = 'kafka-pod:9092';
5+
let kafkaTopic = 'testingtopic';
6+
7+
try {
8+
const Producer = kafka.Producer;
9+
const client = new kafka.KafkaClient({kafkaHost: kafkaHost});
10+
const producer = new Producer(client);
11+
let payloads = [
12+
{
13+
topic: kafkaTopic,
14+
messages: 'Hello world!',
15+
partition: 0,
16+
attributes: 0
17+
}
18+
];
19+
20+
producer.on('ready', function() {
21+
producer.send(payloads, (err: any, data: any) => {
22+
if (err) {
23+
console.log('[kafka-producer -> '+kafkaTopic+']: broker update failed');
24+
} else {
25+
console.log('[kafka-producer -> '+kafkaTopic+']: broker update success');
26+
}
27+
});
28+
});
29+
30+
producer.on('error', function(err: any) {
31+
console.log(err);
32+
console.log('[kafka-producer -> '+kafkaTopic+']: connection errored');
33+
throw err;
34+
});
35+
}
36+
catch(e) {
37+
console.log(e);
38+
}

‎kafka-pubsub/project/tsconfig.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"compilerOptions": {
3+
/* Visit https://aka.ms/tsconfig.json to read more about this file */
4+
5+
/* Basic Options */
6+
// "incremental": true, /* Enable incremental compilation */
7+
"target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
8+
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
9+
// "lib": [], /* Specify library files to be included in the compilation. */
10+
// "allowJs": true, /* Allow javascript files to be compiled. */
11+
// "checkJs": true, /* Report errors in .js files. */
12+
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
13+
// "declaration": true, /* Generates corresponding '.d.ts' file. */
14+
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
15+
// "sourceMap": true, /* Generates corresponding '.map' file. */
16+
// "outFile": "./", /* Concatenate and emit output to single file. */
17+
"outDir": "./build", /* Redirect output structure to the directory. */
18+
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
19+
// "composite": true, /* Enable project compilation */
20+
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
21+
// "removeComments": true, /* Do not emit comments to output. */
22+
// "noEmit": true, /* Do not emit outputs. */
23+
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
24+
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
25+
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
26+
27+
/* Strict Type-Checking Options */
28+
"strict": true, /* Enable all strict type-checking options. */
29+
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
30+
// "strictNullChecks": true, /* Enable strict null checks. */
31+
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
32+
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
33+
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
34+
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
35+
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
36+
37+
/* Additional Checks */
38+
// "noUnusedLocals": true, /* Report errors on unused locals. */
39+
// "noUnusedParameters": true, /* Report errors on unused parameters. */
40+
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
41+
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
42+
43+
/* Module Resolution Options */
44+
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
45+
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
46+
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
47+
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
48+
// "typeRoots": [], /* List of folders to include type definitions from. */
49+
// "types": [], /* Type declaration files to be included in compilation. */
50+
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
51+
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
52+
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
53+
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
54+
55+
/* Source Map Options */
56+
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
57+
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
58+
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
59+
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
60+
61+
/* Experimental Options */
62+
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
63+
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
64+
65+
/* Advanced Options */
66+
"skipLibCheck": true, /* Skip type checking of declaration files. */
67+
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
68+
}
69+
}

0 commit comments

Comments
 (0)