From 9cfe5f639ff90d685e8f8cbd1d7556e2c2c52495 Mon Sep 17 00:00:00 2001
From: "Heeill(Floyd)Wang"
Date: Fri, 10 Aug 2018 16:54:13 +0200
Subject: [PATCH 01/41] test calculator
---
app/conf/plugins.json | 12 ++++++++++++
app/plugins/tutorial/calculator.html | 4 ++++
app/plugins/tutorial/calculator.js | 7 +++++++
app/plugins/tutorial/tutorial.html | 4 ++++
4 files changed, 27 insertions(+)
mode change 100644 => 100755 app/conf/plugins.json
create mode 100644 app/plugins/tutorial/calculator.html
create mode 100644 app/plugins/tutorial/calculator.js
mode change 100644 => 100755 app/plugins/tutorial/tutorial.html
diff --git a/app/conf/plugins.json b/app/conf/plugins.json
old mode 100644
new mode 100755
index f53e5a6aa..208075ef1
--- a/app/conf/plugins.json
+++ b/app/conf/plugins.json
@@ -53,4 +53,16 @@
"controller": "ShardsController"
}
}
+}, {
+ "name": "calculator",
+ "uri": "static/plugins/tutorial/calculator.js",
+ "enterprise": false,
+ "routing": {
+ "/calculator": {
+ "name": "calculator",
+ "url": "/calculator",
+ "templateUrl": "static/plugins/tutorial/calculator.html",
+ "controller": "CalculatorController"
+ }
+ }
}]
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
new file mode 100644
index 000000000..ce767f2a1
--- /dev/null
+++ b/app/plugins/tutorial/calculator.html
@@ -0,0 +1,4 @@
+
+ test calculator
+ {{test}}
+
\ No newline at end of file
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
new file mode 100644
index 000000000..6be893744
--- /dev/null
+++ b/app/plugins/tutorial/calculator.js
@@ -0,0 +1,7 @@
+'use strict';
+
+// storage is measured in Bytes
+// time is measured in hours
+angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope) {
+ $scope.test = 'test string';
+});
\ No newline at end of file
diff --git a/app/plugins/tutorial/tutorial.html b/app/plugins/tutorial/tutorial.html
old mode 100644
new mode 100755
index 4f994db9f..0556cb748
--- a/app/plugins/tutorial/tutorial.html
+++ b/app/plugins/tutorial/tutorial.html
@@ -77,6 +77,10 @@ {{:: 'HELP.ENTERPRISE_TITLE' | translate}}
{{:: 'HELP.ENTERPRISE_MSG' | translate }}
+
+
+ calculator
+
From 598d112462030368e6e5dade826931eaeef6a435 Mon Sep 17 00:00:00 2001
From: "Heeill(Floyd)Wang"
Date: Fri, 10 Aug 2018 17:01:52 +0200
Subject: [PATCH 02/41] add .html and .js
---
app/plugins/tutorial/calculator.html | 181 ++++++++++++++++++++++++++-
app/plugins/tutorial/calculator.js | 95 +++++++++++++-
2 files changed, 272 insertions(+), 4 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index ce767f2a1..204ccd10d 100644
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -1,4 +1,179 @@
- test calculator
- {{test}}
-
\ No newline at end of file
+
+
+ The ultimative guide on sharding
+
+
+
+
+Hardware
+
+
+CPUs
+How many cores do the machines have you are going to run your
+nodes on?
+CPUCoresPerNode
+
+
+
+ selected cores/node: {{ CPUCoresPerNode }}
+
+
+
+
+Storage
+You are using SSDs instead of HDDs? Good.
+
+
+
+RAM/Storage
+The proportion of RAM and storage generally influences the performance.
+ A ratio around 1:24 is recommended, if you do not care for speed that much, you can go a little below that,
+ if you do, you can go higher, but astronomical are not really improve performance anymore.
+
+RAMStorageProportion
+
+
+
+ each GB of RAM serves {{RAMStorageProportion}} of storage
+
+
+
+
+RAM
+Each node is expected to have 64GB of RAM, since this is the max without causing major garbage collection issues.
+
+
+
+
+RAID?
+Crate.io is distributed and self healing, replicas of the data are distributed among the cluster, if they are enabled.
+ This is probably something you want, because it makes something like RAID1 superfluous and even enables the use of RAID0 in a safe way.
+
+
+
+
+
+
+Use-Case
+Data
+So, this is what crate is about. You have probably been using
+something else before, so you should generally know how much data
+there is.
+How much is being inserted?
+dataInsertedPerTime
+
+
+
+
+ data insertion amount: {{dataInsertedPerTime}}
+
+
+ Mega
+ Giga
+ Terra
+
+ data insertion unit prefix: {{dataInsertedPerTimeUnitPrefix}}
+
+
+ hour
+ day
+ week
+ month
+
+ data insertion time unit: {{dataInsertedPerTimeTemporalUnit}}
+
+
+ keep time: {{keepTime}}
+
+
+ hour
+ day
+ week
+ month
+ year
+
+ keep time unit: {{keepTimeTemporalUnit}}
+
+How long is it supposed to be stored?
+keepTime
+
+
+
+Since crate is horizontally scalable you can also later decide that
+the data is supposed to be stored for longer, by adding nodes.
+OR, if it’s really not known yet, just specify some expected
+table size.
+expectedTableSize
+
+
+
+
+ expected table size: {{expectedTableSize}}
+
+
+ Mega
+ Giga
+ Terra
+
+ expected table size unit prefix: {{expectedTableSizeUnitPrefix}}
+
+
+Partitioning
+Crate organizes data by itself into logical units, usually by
+time. If a good time frame for partitioning is set, this can enhance
+the performance greatly. Choose a batch size in which the data will
+be used later. If you have no idea, a month is a good sugestion.
+partionSize
+
+
+
+ partitionSize: {{partitionSize}}
+
+
+ hour
+ day
+ week
+ month
+ year
+
+ partitionSizeTemporalUnit: {{partitionSizeTemporalUnit}}
+
+
+
+ You have expected table size selected, in this case you need to specify the amount of partitions manually.
+
+ partitionCount: {{manualPartitionCount}}
+
+
+Redundancy
+As said crate distributes replicas of the data among its network,
+one replica should be affordable storage wise, for redundancy and
+data integrity. Having no replicas is of course easy on storage but
+not advisable. By having more replicas the query speed can be
+enhanced significantly, because one query then can run on several
+copies of the same data simultaneously. Sadly that is very storage
+intense and does affect the write speed a little. So one replica is
+the normal way to go if you are very into query speed you can go with
+3-4. Aside from that, having more replicas than nodes never makes
+sense. They have nowhere to go.
+replicas
+
+
+
+ replicas: {{replicas}}
+
+
+
+
+
+
+
+Calculation
+
+ By the data provided, this little script recommends to split your table into: shards.
+
+
+
+
+
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
index 6be893744..cdeafdcdd 100644
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -3,5 +3,98 @@
// storage is measured in Bytes
// time is measured in hours
angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope) {
- $scope.test = 'test string';
+ $scope.diskLoadFactor = 0.85;
+ $scope.maxRAMPerNode = 64000000000; //64G
+ $scope.sizeFactor = 0.732; //from haudi's document
+ $scope.maxShardSize = 20000000000; //20G
+ $scope.maxShards = 1000;
+ $scope.CPUCoresPerNode='2';
+ $scope.RAMStorageProportion='24';
+ $scope.dataType='perTime';
+ $scope.dataInsertedPerTime='10';
+ $scope.expectedTableSize='10';
+ $scope.expectedTableSizeUnitPrefix='Giga';
+ $scope.dataInsertedPerTimeUnitPrefix='Giga';
+ $scope.dataInsertedPerTimeTemporalUnit='day';
+ $scope.expectedTableSize='10';
+ $scope.keepTimeTemporalUnit='month';
+ $scope.keepTime='6';
+ $scope.partitionSize='1';
+ $scope.partitionSizeTemporalUnit='month';
+ $scope.manualPartitionCount = 4;
+ $scope.replicas='1';
+ $scope.neededDiskSpace = function() {
+ var res;
+ if ($scope.dataType === 'absolute') {
+ res = $scope.expectedTableSize * $scope.prefix($scope.expectedTableSizeUnitPrefix) / $scope.sizeFactor / $scope.diskLoadFactor;
+ } else if ($scope.dataType === 'perTime') {
+ res = (($scope.prefix($scope.dataInsertedPerTimeUnitPrefix) * $scope.dataInsertedPerTime / $scope.temporalUnit($scope.dataInsertedPerTimeTemporalUnit)) * $scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit) * (1 + $scope.replicas)) / $scope.diskLoadFactor / $scope.sizeFactor;
+ }
+ console.log("neededDiskSpace() returning: " + res);
+ return res;
+ };
+ $scope.neededNodes = function() {
+ var res = Math.ceil(($scope.neededDiskSpace() / $scope.RAMStorageProportion) / $scope.maxRAMPerNode);
+ console.log("neededNodes() returning: " + res);
+ return res;
+ };
+ $scope.partitions = function() {
+ var r;
+ if($scope.dataType === 'perTime') {
+ r = (($scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit)) / ($scope.partitionSize * $scope.temporalUnit($scope.partitionSizeTemporalUnit)));
+ }else {
+ r = $scope.manualPartitionCount;
+ }
+ console.log("partitionCount: " + r);
+ return r;
+ };
+ $scope.shards = function() {
+ var res = Math.ceil(($scope.neededNodes() * $scope.CPUCoresPerNode) / $scope.partitions());
+ console.log("shards() returning: " + res);
+ return res;
+ };
+ $scope.shardSize = function(shards) {
+ var res = $scope.neededDiskSpace / ((($scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit)) / $scope.partitions()) * shards * $scope.replicas);
+ };
+ $scope.prefix = function(x) {
+ switch (x) {
+ case "Terra":
+ return Math.pow(10, 12);
+ case "Giga":
+ return Math.pow(10, 9);
+ case "Mega":
+ return Math.pow(10, 6);
+ case "Kilo":
+ return Math.pow(10, 3);
+ default:
+ return Math.pow(10, 0);
+ }
+ };
+ $scope.temporalUnit = function(x) {
+ switch (x) {
+ case "hour":
+ return 1;
+ case "day":
+ return 24;
+ case "week":
+ return 7 * 24;
+ case "month":
+ return 30 * 24;
+ case "year":
+ return 356 * 24;
+ default:
+ return 1;
+ }
+ };
+ $scope.result = function () {
+ var s = shards();
+ if(s > $scope.maxShards){
+ return "maximum shard limit exceeded, please talk to an crate engineer about your use-case";
+ }
+ while ($scope.shardSize(s) > $scope.maxShardSize){
+ s++;
+ }
+ return s;
+ };
+
});
\ No newline at end of file
From 64f8300099a5f60cf72077ecbaa16db7245bc5eb Mon Sep 17 00:00:00 2001
From: "Heeill(Floyd)Wang"
Date: Fri, 10 Aug 2018 17:13:09 +0200
Subject: [PATCH 03/41] calculator.js : put function
---
app/plugins/tutorial/calculator.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
index cdeafdcdd..7ba7ec895 100644
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -2,7 +2,7 @@
// storage is measured in Bytes
// time is measured in hours
-angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope) {
+angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope, SQLQuery) {
$scope.diskLoadFactor = 0.85;
$scope.maxRAMPerNode = 64000000000; //64G
$scope.sizeFactor = 0.732; //from haudi's document
From d99493d8d98afa193ae7c2836052066e0ace7661 Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Fri, 10 Aug 2018 17:25:44 +0200
Subject: [PATCH 04/41] change design
---
app/plugins/tutorial/calculator.html | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
mode change 100644 => 100755 app/plugins/tutorial/calculator.html
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
old mode 100644
new mode 100755
index 204ccd10d..f38526364
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -56,6 +56,9 @@ RAID?
Use-Case
+
+
+
Data
So, this is what crate is about. You have probably been using
something else before, so you should generally know how much data
@@ -65,7 +68,7 @@
Data
-
+
data insertion amount: {{dataInsertedPerTime}}
@@ -83,7 +86,7 @@ Data
data insertion time unit: {{dataInsertedPerTimeTemporalUnit}}
-
+
keep time: {{keepTime}}
@@ -104,11 +107,11 @@ Data
the data is supposed to be stored for longer, by adding nodes.
OR, if it’s really not known yet, just specify some expected
table size.
-expectedTableSize
+expectedTableSize
-
+
expected table size: {{expectedTableSize}}
@@ -119,15 +122,20 @@ Data
expected table size unit prefix: {{expectedTableSizeUnitPrefix}}
-Partitioning
+
+
+
+
+
+Partitioning
Crate organizes data by itself into logical units, usually by
time. If a good time frame for partitioning is set, this can enhance
the performance greatly. Choose a batch size in which the data will
be used later. If you have no idea, a month is a good sugestion.
-partionSize
+partionSize
-
+
partitionSize: {{partitionSize}}
@@ -142,11 +150,11 @@ Data
You have expected table size selected, in this case you need to specify the amount of partitions manually.
-
+
partitionCount: {{manualPartitionCount}}
-Redundancy
+Redundancy
As said crate distributes replicas of the data among its network,
one replica should be affordable storage wise, for redundancy and
data integrity. Having no replicas is of course easy on storage but
@@ -160,7 +168,7 @@
Data
replicas
-
+
replicas: {{replicas}}
From c32466e441bf57277797c9413628bb9993f36444 Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Mon, 13 Aug 2018 15:26:20 +0200
Subject: [PATCH 05/41] change shard calculator to newest version
---
app/plugins/tutorial/calculator.html | 416 +++++++++++++++------------
app/plugins/tutorial/calculator.js | 85 +++---
2 files changed, 274 insertions(+), 227 deletions(-)
mode change 100644 => 100755 app/plugins/tutorial/calculator.js
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index f38526364..1f31cc9d5 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -1,187 +1,229 @@
-
-
-
- The ultimative guide on sharding
-
-
-
-
-
Hardware
-
-
-
CPUs
-
How many cores do the machines have you are going to run your
-nodes on?
-
CPUCoresPerNode
-
-
-
- selected cores/node: {{ CPUCoresPerNode }}
-
-
-
-
-
Storage
-
You are using SSDs instead of HDDs? Good.
-
-
-
-
RAM/Storage
-
The proportion of RAM and storage generally influences the performance.
- A ratio around 1:24 is recommended, if you do not care for speed that much, you can go a little below that,
- if you do, you can go higher, but astronomical are not really improve performance anymore.
-
-
RAMStorageProportion
-
-
-
- each GB of RAM serves {{RAMStorageProportion}} of storage
-
-
-
-
-
RAM
-
Each node is expected to have 64GB of RAM, since this is the max without causing major garbage collection issues.
-
-
-
-
-
RAID?
-
Crate.io is distributed and self healing, replicas of the data are distributed among the cluster, if they are enabled.
- This is probably something you want, because it makes something like RAID1 superfluous and even enables the use of RAID0 in a safe way.
-
-
-
-
-
-
-
Use-Case
-
-
-
-
Data
-
So, this is what crate is about. You have probably been using
-something else before, so you should generally know how much data
-there is.
-
How much is being inserted?
-
dataInsertedPerTime
-
-
-
-
- data insertion amount: {{dataInsertedPerTime}}
-
-
- Mega
- Giga
- Terra
-
- data insertion unit prefix: {{dataInsertedPerTimeUnitPrefix}}
-
-
- hour
- day
- week
- month
-
- data insertion time unit: {{dataInsertedPerTimeTemporalUnit}}
-
-
- keep time: {{keepTime}}
-
-
- hour
- day
- week
- month
- year
-
- keep time unit: {{keepTimeTemporalUnit}}
-
-How long is it supposed to be stored?
-keepTime
-
-
-
-Since crate is horizontally scalable you can also later decide that
-the data is supposed to be stored for longer, by adding nodes.
-
OR, if it’s really not known yet, just specify some expected
-table size.
-
expectedTableSize
-
-
-
-
- expected table size: {{expectedTableSize}}
-
-
- Mega
- Giga
- Terra
-
- expected table size unit prefix: {{expectedTableSizeUnitPrefix}}
-
-
-
-
-
-
-
-
Partitioning
-
Crate organizes data by itself into logical units, usually by
-time. If a good time frame for partitioning is set, this can enhance
-the performance greatly. Choose a batch size in which the data will
-be used later. If you have no idea, a month is a good sugestion.
-
partionSize
-
-
-
- partitionSize: {{partitionSize}}
-
-
- hour
- day
- week
- month
- year
-
- partitionSizeTemporalUnit: {{partitionSizeTemporalUnit}}
-
-
-
- You have expected table size selected, in this case you need to specify the amount of partitions manually.
-
- partitionCount: {{manualPartitionCount}}
-
-
-
Redundancy
-
As said crate distributes replicas of the data among its network,
-one replica should be affordable storage wise, for redundancy and
-data integrity. Having no replicas is of course easy on storage but
-not advisable. By having more replicas the query speed can be
-enhanced significantly, because one query then can run on several
-copies of the same data simultaneously. Sadly that is very storage
-intense and does affect the write speed a little. So one replica is
-the normal way to go if you are very into query speed you can go with
-3-4. Aside from that, having more replicas than nodes never makes
-sense. They have nowhere to go.
-
replicas
-
-
-
- replicas: {{replicas}}
-
-
-
-
-
-
-
-
Calculation
-
- By the data provided, this little script recommends to split your table into: shards.
-
-
-
-
-
+
+
+
+
+ Sharding calculator
+
+
+
+
+
+
+
+
+
+
+ Sharding calculator
+
+
+ A simple tool to calculate some generic sharding recommendation for a crate table.
+
+
+ Hardware
+
+
+ CPUs
+
+
+ How many cores do the machines have you are going to run your
+ nodes on?
+
+
+
+
+
+
+
+ Storage
+
+
+ You are using SSDs instead of HDDs? Good.
+
+
+
+ RAM/Storage
+
+
+ The proportion of RAM and storage generally influences the
+ performance. A ratio around 1:24 is recommended, if you do not care
+ for speed that much, you can go a little below that, if you do, you
+ can go higher, but astronomical are not really improve performance
+ anymore.
+
+
+
+
+ Every GB of RAM serves
+
+
+
+ GB of storage.
+
+
+
+
+ RAM
+
+
Each node is expected to have 64GB of RAM, since this is the max without causing major garbage collection
+ issues.
+
+
+
+ RAID?
+
+
+ Crate.io is distributed and self healing, replicas of the data are
+ distributed among the cluster, if they are enabled. This is probably
+ something you want, because it makes something like RAID1 superfluous
+ and even enables the use of RAID0 in a safe way.
+
+
+ Use-Case
+
+
+ Data
+
+
+ So, this is what crate is about. You have probably been using
+ something else before, so you should generally know how much data
+ there is.
+
+
+
+
+
+ How much is being inserted?
+
+
+
+
+
+
+
+ Mega
+ Giga
+ Terra
+
+
+
+ Byte per
+
+
+
+ hour
+ day
+ week
+ month
+
+
+
+ And for how long is it supposed to be stored?
+
+
+
+
+
+ hour
+ day
+ week
+ month
+ year
+
+
+
+
+
+
+ Since crate is horizontally scalable you can also later decide that
+ the data is supposed to be stored for longer, by adding nodes.
+
+
+
+
+
+ OR, if it’s really not known yet, just specify some expected
+ table size.
+
+
+
+
+
+
+
+ Mega
+ Giga
+ Terra
+
+
+
+ Byte
+
+
+
+
+
+
+ Partitioning
+
+
+ Crate organizes data by itself into logical units, usually by
+ time. If a good time frame for partitioning is set, this can enhance
+ the performance greatly. Choose a batch size in which the data will
+ be used later. If you have no idea, a month is a good suggestion.
+
+
+
+
+
+
+
+ hour
+ day
+ week
+ month
+ year
+
+
+
+
+
+ You have expected table size selected, in this case you need to specify
+ the amount of partitions manually.
+
+
+
+
+
+
+ Redundancy
+
+
+ As said crate distributes replicas of the data among its network,
+ one replica should be affordable storage wise, for redundancy and
+ data integrity. Having no replicas is of course easy on storage but
+ not advisable. By having more replicas the query speed can be
+ enhanced significantly, because one query then can run on several
+ copies of the same data simultaneously. Sadly that is very storage
+ intense and does affect the write speed a little. So one replica is
+ the normal way to go if you are very into query speed you can go with
+ 3-4. Aside from that, having more replicas than nodes never makes
+ sense. They have nowhere to go.
+
+
+
+
+
+
+
+ Calculation
+
+
+
+ By the data provided, this little script recommends to split your table into:
+
+ shards.
+
+
+
+
+
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
old mode 100644
new mode 100755
index 7ba7ec895..b572b907a
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -3,60 +3,66 @@
// storage is measured in Bytes
// time is measured in hours
angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope, SQLQuery) {
+ // storage is measured in Bytes
+ // time is measured in hours
$scope.diskLoadFactor = 0.85;
$scope.maxRAMPerNode = 64000000000; //64G
$scope.sizeFactor = 0.732; //from haudi's document
$scope.maxShardSize = 20000000000; //20G
$scope.maxShards = 1000;
- $scope.CPUCoresPerNode='2';
- $scope.RAMStorageProportion='24';
- $scope.dataType='perTime';
- $scope.dataInsertedPerTime='10';
- $scope.expectedTableSize='10';
- $scope.expectedTableSizeUnitPrefix='Giga';
- $scope.dataInsertedPerTimeUnitPrefix='Giga';
- $scope.dataInsertedPerTimeTemporalUnit='day';
- $scope.expectedTableSize='10';
- $scope.keepTimeTemporalUnit='month';
- $scope.keepTime='6';
- $scope.partitionSize='1';
- $scope.partitionSizeTemporalUnit='month';
+ $scope.CPUCoresPerNode = '2';
+ $scope.RAMStorageProportion = '24';
+ $scope.dataType = 'perTime';
+ $scope.dataInsertedPerTime = '100';
+ $scope.expectedTableSize = '10';
+ $scope.expectedTableSizeUnitPrefix = 'Terra';
+ $scope.dataInsertedPerTimeUnitPrefix = 'Giga';
+ $scope.dataInsertedPerTimeTemporalUnit = 'day';
+ $scope.expectedTableSize = '10';
+ $scope.keepTimeTemporalUnit = 'month';
+ $scope.keepTime = '6';
+ $scope.partitionSize = '1';
+ $scope.partitionSizeTemporalUnit = 'month';
$scope.manualPartitionCount = 4;
- $scope.replicas='1';
- $scope.neededDiskSpace = function() {
- var res;
+ $scope.replicas = '1';
+ $scope.neededDiskSpace = function () {
+ var res = 1;
if ($scope.dataType === 'absolute') {
- res = $scope.expectedTableSize * $scope.prefix($scope.expectedTableSizeUnitPrefix) / $scope.sizeFactor / $scope.diskLoadFactor;
+ res = ($scope.expectedTableSize * $scope.prefix($scope.expectedTableSizeUnitPrefix) * (1 + Number($scope.replicas))) / $scope.sizeFactor / $scope.diskLoadFactor;
} else if ($scope.dataType === 'perTime') {
- res = (($scope.prefix($scope.dataInsertedPerTimeUnitPrefix) * $scope.dataInsertedPerTime / $scope.temporalUnit($scope.dataInsertedPerTimeTemporalUnit)) * $scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit) * (1 + $scope.replicas)) / $scope.diskLoadFactor / $scope.sizeFactor;
+ res = (($scope.prefix($scope.dataInsertedPerTimeUnitPrefix) * $scope.dataInsertedPerTime / $scope.temporalUnit($scope.dataInsertedPerTimeTemporalUnit)) * $scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit) * (1 + Number($scope.replicas))) / $scope.diskLoadFactor / $scope.sizeFactor; //explicit cast of replica to number is necessary, otherwise 1+1=11. thanks java script
}
- console.log("neededDiskSpace() returning: " + res);
+ console.log("neededDiskSpace: " + Math.round(res / Math.pow(10, 9)) + "GB");
return res;
};
- $scope.neededNodes = function() {
+ $scope.neededNodes = function () {
var res = Math.ceil(($scope.neededDiskSpace() / $scope.RAMStorageProportion) / $scope.maxRAMPerNode);
- console.log("neededNodes() returning: " + res);
+ console.log("neededNodes: " + res);
return res;
};
- $scope.partitions = function() {
- var r;
- if($scope.dataType === 'perTime') {
- r = (($scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit)) / ($scope.partitionSize * $scope.temporalUnit($scope.partitionSizeTemporalUnit)));
- }else {
- r = $scope.manualPartitionCount;
+ $scope.partitions = function () {
+ var res = 1;
+ if ($scope.dataType === 'perTime') {
+ res = (($scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit)) / ($scope.partitionSize * $scope.temporalUnit($scope.partitionSizeTemporalUnit)));
+ } else if ($scope.dataType === 'perTime') {
+ res = $scope.manualPartitionCount;
}
- console.log("partitionCount: " + r);
- return r;
+ console.log("partitionCount: " + res);
+ return res;
};
- $scope.shards = function() {
+ $scope.shards = function () {
var res = Math.ceil(($scope.neededNodes() * $scope.CPUCoresPerNode) / $scope.partitions());
- console.log("shards() returning: " + res);
+ console.log("shards(): " + res);
return res;
};
- $scope.shardSize = function(shards) {
- var res = $scope.neededDiskSpace / ((($scope.keepTime * $scope.temporalUnit($scope.keepTimeTemporalUnit)) / $scope.partitions()) * shards * $scope.replicas);
+ $scope.shardSize = function (shards) {
+ console.log("replicas: " + (1 + Number($scope.replicas)));
+
+ var res = $scope.neededDiskSpace() / (shards * $scope.partitions() * (1 + Number($scope.replicas)));
+ console.log("shardSize: " + res);
+ return res;
};
- $scope.prefix = function(x) {
+ $scope.prefix = function (x) {
switch (x) {
case "Terra":
return Math.pow(10, 12);
@@ -70,7 +76,7 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
return Math.pow(10, 0);
}
};
- $scope.temporalUnit = function(x) {
+ $scope.temporalUnit = function (x) {
switch (x) {
case "hour":
return 1;
@@ -81,20 +87,19 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
case "month":
return 30 * 24;
case "year":
- return 356 * 24;
+ return 365 * 24;
default:
return 1;
}
};
$scope.result = function () {
- var s = shards();
- if(s > $scope.maxShards){
+ var s = $scope.shards();
+ if (s > $scope.maxShards) {
return "maximum shard limit exceeded, please talk to an crate engineer about your use-case";
}
- while ($scope.shardSize(s) > $scope.maxShardSize){
+ while ($scope.shardSize(s) > $scope.maxShardSize) {
s++;
}
return s;
};
-
});
\ No newline at end of file
From b458bf1f496404fd781262b12bb09d029f70df96 Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Mon, 13 Aug 2018 17:00:37 +0200
Subject: [PATCH 06/41] correct error in shard calculator
---
app/plugins/tutorial/calculator.html | 32 ++++++++++++++--------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index 1f31cc9d5..3c95e087e 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -10,7 +10,7 @@
-
+
Sharding calculator
@@ -30,7 +30,7 @@
-
+
@@ -55,7 +55,7 @@
Every GB of RAM serves
-
+
GB of storage.
@@ -91,15 +91,15 @@
-
+
How much is being inserted?
-
+
-
+
Mega
Giga
Terra
@@ -109,7 +109,7 @@
Byte per
-
+
hour
day
week
@@ -120,9 +120,9 @@
And for how long is it supposed to be stored?
-
+
-
+
hour
day
week
@@ -140,16 +140,16 @@
-
+
OR, if it’s really not known yet, just specify some expected
table size.
-
+
-
+
Mega
Giga
Terra
@@ -174,9 +174,9 @@
-
+
-
+
hour
day
week
@@ -190,7 +190,7 @@
You have expected table size selected, in this case you need to specify
the amount of partitions manually.
-
+
@@ -211,7 +211,7 @@
-
+
From 892e16e0e9b6f364643ad38e2c6b4c2cdf4f13bd Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Mon, 13 Aug 2018 18:22:52 +0200
Subject: [PATCH 07/41] correct error in shard calculator
---
app/plugins/tutorial/calculator.html | 52 +++++++++++++++-------------
1 file changed, 27 insertions(+), 25 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index 3c95e087e..c6fb3bc46 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -30,7 +30,7 @@
-
+
@@ -55,7 +55,7 @@
Every GB of RAM serves
-
+
GB of storage.
@@ -91,15 +91,15 @@
-
+
How much is being inserted?
-
+
-
+
Mega
Giga
Terra
@@ -109,7 +109,7 @@
Byte per
-
+
hour
day
week
@@ -120,9 +120,9 @@
And for how long is it supposed to be stored?
-
+
-
+
hour
day
week
@@ -140,16 +140,16 @@
-
+
OR, if it’s really not known yet, just specify some expected
table size.
-
+
-
+
Mega
Giga
Terra
@@ -168,29 +168,31 @@
Crate organizes data by itself into logical units, usually by
time. If a good time frame for partitioning is set, this can enhance
- the performance greatly. Choose a batch size in which the data will
- be used later. If you have no idea, a month is a good suggestion.
+ the performance greatly.
-
-
-
- hour
- day
- week
- month
- year
-
+
+ Choose a batch size in which the data will be used later. If you have no idea, a month is a good suggestion.
+
+
+
+ hour
+ day
+ week
+ month
+ year
+
+
You have expected table size selected, in this case you need to specify
- the amount of partitions manually.
+ the amount of partitions manually
+ partitions.
-
@@ -211,7 +213,7 @@
-
+
From 26f2aa573fc27c043f0f33b807a2f55f78efb1c7 Mon Sep 17 00:00:00 2001
From: leo
Date: Tue, 14 Aug 2018 15:50:11 +0200
Subject: [PATCH 08/41] apply sharding-calculator storage recommendation update
---
app/plugins/tutorial/calculator.html | 24 ++++++++++++++++++++++--
app/plugins/tutorial/calculator.js | 25 +++++++++++++++++++++++++
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index c6fb3bc46..dc99b2d69 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -221,11 +221,31 @@
- By the data provided, this little script recommends to split your table into:
+ With the data provided this little script came up with a recommendation.
+
+
+ nodes
+
+
+ You should use
+
+ nodes with
+
+ of RAM and
+
+ of storage each.
+
+
+
+ shards
+
+
+ The partitions should be divided into
- shards.
+ shards each.
+
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
index b572b907a..6a1eabc22 100755
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -62,6 +62,31 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
console.log("shardSize: " + res);
return res;
};
+ $scope.ramString = function () {
+ var res = $scope.RAMStorageProportion + "GB";
+ console.log("ramString: " + res);
+ return res;
+ };
+ $scope.storageString = function () {
+ return $scope.bytesToPrintableString(Math.round(Number($scope.neededDiskSpace()) / Number($scope.neededNodes())));
+ };
+ $scope.bytesToPrintableString = function (b) {
+ console.log("entering bytify...");
+ var strg;
+ if (b < Math.pow(10, 3)) {
+ strg = b + "B";
+ } else if (b < Math.pow(10, 6)){
+ strg = (b / Math.pow(10, 3)).toFixed(2) + "KB";
+ } else if (b < Math.pow(10, 9)){
+ strg = (b / Math.pow(10, 6)).toFixed(2) + "MB";
+ } else if (b < Math.pow(10, 12)) {
+ strg = (b / Math.pow(10, 9)).toFixed(2) + "GB";
+ } else {
+ strg = (b / Math.pow(10, 12)).toFixed(2) + "TB"
+ }
+ console.log("bytify: " + strg);
+ return strg;
+ };
$scope.prefix = function (x) {
switch (x) {
case "Terra":
From e10037c18d1b6fad31b20e5ae81bc10706673379 Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Thu, 16 Aug 2018 00:28:23 +0200
Subject: [PATCH 09/41] select schema, table from database and get it's name as
string in JavaScript /testing
---
app/conf/plugins.json | 12 +++++++++
app/plugins/tutorial/testing.html | 21 ++++++++++++++++
app/plugins/tutorial/testing.js | 42 +++++++++++++++++++++++++++++++
3 files changed, 75 insertions(+)
create mode 100644 app/plugins/tutorial/testing.html
create mode 100644 app/plugins/tutorial/testing.js
diff --git a/app/conf/plugins.json b/app/conf/plugins.json
index 208075ef1..992c79024 100755
--- a/app/conf/plugins.json
+++ b/app/conf/plugins.json
@@ -65,4 +65,16 @@
"controller": "CalculatorController"
}
}
+}, {
+ "name": "testing",
+ "uri": "static/plugins/tutorial/testing.js",
+ "enterprise": false,
+ "routing": {
+ "/testing": {
+ "name": "testing",
+ "url": "/testing",
+ "templateUrl": "static/plugins/tutorial/testing.html",
+ "controller": "testingController"
+ }
+ }
}]
diff --git a/app/plugins/tutorial/testing.html b/app/plugins/tutorial/testing.html
new file mode 100644
index 000000000..9980fefae
--- /dev/null
+++ b/app/plugins/tutorial/testing.html
@@ -0,0 +1,21 @@
+
+
choose your table name
+
+
+
+ SCHEMA:
+
+
+ OK
+
+
+
+ TABLE:
+
+
+ OK
+
+
+
+ {{selectedSchema}} {{selectedTable}}
+
\ No newline at end of file
diff --git a/app/plugins/tutorial/testing.js b/app/plugins/tutorial/testing.js
new file mode 100644
index 000000000..aef414dab
--- /dev/null
+++ b/app/plugins/tutorial/testing.js
@@ -0,0 +1,42 @@
+'use strict';
+
+angular.module('testing', ['sql', 'translation']).controller('testingController', function($scope, $window, SQLQuery, queryResultToObjects) {
+
+ $scope.selectedSchema = "";
+ $scope.selectedTable = "";
+ var flag1 = "0";
+ $scope.getschema = function() {
+ if (flag1=="0"){
+ var stmt = 'select schema_name from information_schema.schemata order by schema_name;'
+ var cols = ['schema_name'];
+ var obj = [];
+ SQLQuery.execute(stmt, {}, false, false, false, false)
+ .then(function (query) {
+ $scope.sqlresult = queryResultToObjects(query, cols);
+ })
+ for(var i=0; i<$scope.sqlresult.length; i++) {
+ obj.push($scope.sqlresult[i].schema_name);
+ }
+ $scope.schemaList = obj;
+ flag1="1";
+ }
+ };
+ var flag2 = "0";
+ $scope.getTablename = function(num1) {
+ if (flag2=="0" || num1 =="1"){
+ var stmt = "SELECT table_name FROM information_schema.tables WHERE table_schema ='"+$scope.selectedSchema+"';";
+ var cols = ['table_name'];
+ var obj = [];
+ SQLQuery.execute(stmt, {}, false, false, false, false)
+ .then(function (query) {
+ $scope.sqlresult2 = queryResultToObjects(query, cols);
+ })
+ for(var i=0; i<$scope.sqlresult2.length; i++) {
+ obj.push($scope.sqlresult2[i].table_name);
+ }
+ $scope.tableList = obj;
+ flag2="1";
+ }
+ };
+
+});
\ No newline at end of file
From 177e25c0e45d6877e49b1179e3da1b190acdcd7d Mon Sep 17 00:00:00 2001
From: leo
Date: Thu, 16 Aug 2018 15:17:49 +0200
Subject: [PATCH 10/41] implement querying of host cpu cores
---
app/plugins/tutorial/calculator.html | 17 ++++++++++++++---
app/plugins/tutorial/calculator.js | 22 +++++++++++++++++-----
2 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index dc99b2d69..57cda178b 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -15,9 +15,20 @@
Sharding calculator
-
- A simple tool to calculate some generic sharding recommendation for a crate table.
-
+
+
+
+
+ A simple tool to calculate some generic sharding recommendation for a crate table.
+
+
+
+
+ read from existing table
+
+
+
+
Hardware
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
index 6a1eabc22..d03f8fab6 100755
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -2,9 +2,8 @@
// storage is measured in Bytes
// time is measured in hours
-angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope, SQLQuery) {
- // storage is measured in Bytes
- // time is measured in hours
+
+angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope, SQLQuery, queryResultToObjects) {
$scope.diskLoadFactor = 0.85;
$scope.maxRAMPerNode = 64000000000; //64G
$scope.sizeFactor = 0.732; //from haudi's document
@@ -13,7 +12,7 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
$scope.CPUCoresPerNode = '2';
$scope.RAMStorageProportion = '24';
$scope.dataType = 'perTime';
- $scope.dataInsertedPerTime = '100';
+ $scope.dataInsertedPerTime = '20';
$scope.expectedTableSize = '10';
$scope.expectedTableSizeUnitPrefix = 'Terra';
$scope.dataInsertedPerTimeUnitPrefix = 'Giga';
@@ -25,6 +24,8 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
$scope.partitionSizeTemporalUnit = 'month';
$scope.manualPartitionCount = 4;
$scope.replicas = '1';
+ $scope.tables = ["table1", "table2"];
+ $scope.selectTable = "none";
$scope.neededDiskSpace = function () {
var res = 1;
if ($scope.dataType === 'absolute') {
@@ -71,7 +72,6 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
return $scope.bytesToPrintableString(Math.round(Number($scope.neededDiskSpace()) / Number($scope.neededNodes())));
};
$scope.bytesToPrintableString = function (b) {
- console.log("entering bytify...");
var strg;
if (b < Math.pow(10, 3)) {
strg = b + "B";
@@ -127,4 +127,16 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
}
return s;
};
+ $scope.tableSelected = function () {
+ console.log("selected table: " + $scope.selectTable);
+ $scope.loadData($scope.selectTable);
+ };
+ $scope.loadData = function (tableName) {
+ var stmt = "SELECT os_info['available_processors']\n" +
+ "FROM sys.nodes limit 100;";
+ SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) {
+ $scope.CPUCoresPerNode = (query.rows[0])[0]; //we get a 2d array returned
+ });
+ console.log("queried cpu cores per node: " + $scope.CPUCoresPerNode);
+ };
});
\ No newline at end of file
From 6ced847026f97debffb6fc1407adcea0883549d7 Mon Sep 17 00:00:00 2001
From: leo
Date: Thu, 16 Aug 2018 16:10:47 +0200
Subject: [PATCH 11/41] change iterative shardsize calculation to deterministic
---
app/plugins/tutorial/calculator.js | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js
index d03f8fab6..f4826ed50 100755
--- a/app/plugins/tutorial/calculator.js
+++ b/app/plugins/tutorial/calculator.js
@@ -58,7 +58,6 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
};
$scope.shardSize = function (shards) {
console.log("replicas: " + (1 + Number($scope.replicas)));
-
var res = $scope.neededDiskSpace() / (shards * $scope.partitions() * (1 + Number($scope.replicas)));
console.log("shardSize: " + res);
return res;
@@ -119,12 +118,12 @@ angular.module('calculator', ['sql', 'translation']).controller('CalculatorContr
};
$scope.result = function () {
var s = $scope.shards();
+ if ($scope.shardSize(s) > $scope.maxShardSize) {
+ s = Math.ceil(s * ($scope.shardSize(s) / $scope.maxShardSize));
+ }
if (s > $scope.maxShards) {
return "maximum shard limit exceeded, please talk to an crate engineer about your use-case";
}
- while ($scope.shardSize(s) > $scope.maxShardSize) {
- s++;
- }
return s;
};
$scope.tableSelected = function () {
From 528a0523a2e28471edcf25a16ac975ee913b2afd Mon Sep 17 00:00:00 2001
From: Seonghye Han
Date: Fri, 17 Aug 2018 10:34:15 +0200
Subject: [PATCH 12/41] read real table name
---
app/plugins/tutorial/calculator.html | 6 +++---
app/plugins/tutorial/calculator.js | 21 +++++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html
index 57cda178b..bcf1caccb 100755
--- a/app/plugins/tutorial/calculator.html
+++ b/app/plugins/tutorial/calculator.html
@@ -10,7 +10,7 @@
-
+
Sharding calculator
@@ -26,7 +26,7 @@
read from existing table
-
+
@@ -257,6 +257,6 @@
-
+