Skip to content

Commit 330aadb

Browse files
author
Nikita Sokolov
committed
Merge branch 'feature/puppet-yc' into feature/nexmark-query-8
2 parents b48ab9a + 3c04d15 commit 330aadb

44 files changed

Lines changed: 838 additions & 85 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmark/Puppetfile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
mod 'my', local: true
2+
mod 'geoffwilliams-filemagic', '0.6.1'
3+
mod 'puppet-python', '3.0.1'
4+
mod 'puppet-squid', '2.2.2'
5+
mod 'puppet-alternatives', '3.0.0'
6+
mod 'puppetlabs-apt', '7.6.0'
7+
mod 'puppetlabs-concat', '6.1.0'
8+
mod 'puppetlabs-sshkeys_core'
9+
mod 'puppetlabs-stdlib', '6.0.0'
10+
mod 'puppetlabs-vcsrepo', '3.0.0'
11+
mod 'saz-ssh', '6.2.0'

‎benchmark/ansible/flamestream.yml‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,19 @@
11
---
2-
- name: Install Python
3-
hosts: all
4-
gather_facts: false
5-
tasks:
6-
- name: Install Python 2.x
7-
raw: test -e /usr/bin/python || (apt update && apt install -y python-simplejson)
8-
register: test
9-
changed_when: test.stdout
10-
11-
- name: Prepare cluster
2+
- name: Deliver flamestream artifacts
123
hosts: all
134
roles:
14-
- { role: common-packages }
5+
- { role: flamestream-common }
6+
7+
- name: Stop workers
8+
hosts: workers
9+
tasks:
10+
- shell: "cd {{ flamestream_dir }} && bash entrypoint.sh stop"
1511

1612
- name: Setup zookeeper
1713
hosts: manager
1814
roles:
1915
- { role: zookeeper }
2016

21-
- name: Deliver flamestream artifacts
22-
hosts: all
23-
roles:
24-
- { role: flamestream-common }
25-
2617
- hosts: workers
2718
tasks:
2819
- file:
@@ -47,7 +38,7 @@
4738
- name: Run the benchmark
4839
hosts: bench
4940
roles:
50-
- { role: flamestream-bench , bench_class: com.spbsu.flamestream.example.benchmark.BenchStand }
41+
- { role: flamestream-bench , bench_class: com.spbsu.flamestream.example.benchmark.WikiBenchStand }
5142

5243
- name: Destroy flamestream
5344
hosts: workers

‎benchmark/ansible/group_vars/all.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
rate: 50
1+
rate: 400
22
guarantees: AT_MOST_ONCE
33
millis_between_commits: 50
44
validator: Wiki1000Validator
5-
base_dir: /opt
5+
base_dir: "{{ ansible_env.HOME }}"
66

77
flamestream_dir: "{{ base_dir }}/flamestream"
88
zookeeper_dir: "{{ base_dir }}/zookeeper"
99
bench_dir: "{{ base_dir }}/flamestream"
1010
flink_dir: "{{ base_dir }}/flink"
11-
tarball_dir: /opt/src
11+
tarball_dir: "{{ base_dir }}/tarballs"
1212

1313
results_dir: results/{{ lookup("pipe", "git rev-parse --short HEAD") }}/{{ groups['workers'] | length }}/{{ rate }}/{{ guarantees }}/{{ millis_between_commits }}
1414

‎benchmark/ansible/roles/flamestream-bench/tasks/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
- name: Start the flamestream bench
1818
shell: >
1919
java
20-
-Daeron.term.buffer.length=4194304 -Daeron.mtu.length=16384 -Xms500m -Xmx500m -XX:+PrintGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+HeapDumpOnOutOfMemoryError
20+
-Daeron.term.buffer.length=4194304 -Daeron.mtu.length=16384 -Xms500m -Xmx1500m -Xlog:gc,gc+cpu=info::utc -XX:+HeapDumpOnOutOfMemoryError
2121
-cp '{{ flamestream_dir }}/lib/*'
2222
{{ bench_class }}
2323
{{ flamestream_dir }}/bench.conf {{ flamestream_dir }}/deployer.conf > {{ flamestream_dir }}/bench.log 2>&1

‎benchmark/ansible/roles/flamestream-bench/templates/bench.conf.j2‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
benchmark {
22
bench-host = "{{ groups['bench'][0] }}"
3-
input-host = "{{ groups['input'][0] }}"
3+
input-host = "{{ groups['workers'][0] }}"
44
bench-source-port = 4567
55
bench-sink-port = 5678
66

77
wiki-dump-path = "{{ flamestream_dir }}/wiki.xml"
88
sleep-between-docs-ms = {{ rate }}
9-
worker-id = "{{ groups['worker'][0] }}"
9+
worker-id = "{{ groups['workers'][0] }}"
1010
validator = com.spbsu.flamestream.example.benchmark.validators.{{ validator }}
1111
}

‎benchmark/ansible/roles/zookeeper/tasks/main.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,18 @@
2525
src: zoo.cfg.j2
2626
dest: "{{ zookeeper_dir }}/conf/zoo.cfg"
2727

28-
- name: Restart zk
29-
shell: "{{ zookeeper_dir }}/bin/zkServer.sh restart"
28+
- name: Stop zk
29+
shell: "{{ zookeeper_dir }}/bin/zkServer.sh stop"
3030

31-
- name: Clean graphs
32-
shell: "{{ zookeeper_cli }} rmr /graph"
31+
- file:
32+
state: absent
33+
path: "{{ zookeeper_dir }}/data/"
34+
35+
- name: Start zk
36+
shell: "{{ zookeeper_dir }}/bin/zkServer.sh start"
37+
38+
- command: "{{ zookeeper_dir }}/bin/zkServer.sh status"
39+
retries: 5
40+
register: result
41+
until: result is not failed
3342

34-
- name: Clean configs
35-
shell: "{{ zookeeper_cli }} rmr /config"

‎benchmark/ansible/roles/zookeeper/vars/main.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
zookeeper_dir: "{{ base_dir }}/zookeeper"
3-
zookeeper_url: http://www.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/zookeeper-{{ zookeeper_version }}.tar.gz
3+
zookeeper_url: http://www.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz
44
tick_time: 3000
55
zookeeper_data_dir: "{{ zookeeper_dir }}/data"
6-
zookeeper_cli: "{{ zookeeper_dir }}/bin/zkCli.sh -server localhost:{{ zookeeper_client_port }}"
6+
zookeeper_cli: "{{ zookeeper_dir }}/bin/zkCli.sh -server localhost:{{ zookeeper_client_port }}"

‎benchmark/aws/remote.tf‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
variable "access_key" {}
2+
variable "secret_key" {}
3+
variable "key_pair" {}
4+
variable "cluster_size" {
5+
default = 11
6+
}
7+
variable "region" {
8+
default = "us-east-2"
9+
}
10+
11+
provider "aws" {
12+
access_key = "${var.access_key}"
13+
secret_key = "${var.secret_key}"
14+
region = "${var.region}"
15+
}
16+
17+
resource "aws_key_pair" "mega-benchmarker" {
18+
key_name = "mega-benchmarker"
19+
public_key = "${var.key_pair}"
20+
}
21+
22+
resource "aws_vpc" "workers" {
23+
cidr_block = "10.0.0.0/24"
24+
}
25+
26+
resource "aws_internet_gateway" "workers" {
27+
vpc_id = "${aws_vpc.workers.id}"
28+
}
29+
30+
resource "aws_subnet" "workers" {
31+
vpc_id = "${aws_vpc.workers.id}"
32+
cidr_block = "10.0.0.0/24"
33+
map_public_ip_on_launch = true
34+
}
35+
36+
resource "aws_route_table" "workers" {
37+
vpc_id = "${aws_vpc.workers.id}"
38+
39+
route {
40+
cidr_block = "0.0.0.0/0"
41+
gateway_id = "${aws_internet_gateway.workers.id}"
42+
}
43+
}
44+
45+
resource "aws_route_table_association" "public" {
46+
subnet_id = "${aws_subnet.workers.id}"
47+
route_table_id = "${aws_route_table.workers.id}"
48+
}
49+
50+
resource "aws_security_group" "workers" {
51+
vpc_id = "${aws_vpc.workers.id}"
52+
53+
ingress {
54+
from_port = 0
55+
to_port = 0
56+
protocol = "-1"
57+
cidr_blocks = [
58+
"${aws_vpc.workers.cidr_block}"]
59+
}
60+
61+
ingress {
62+
from_port = 8081
63+
to_port = 8081
64+
protocol = "tcp"
65+
cidr_blocks = [
66+
"0.0.0.0/0"]
67+
}
68+
69+
ingress {
70+
from_port = 22
71+
to_port = 22
72+
protocol = "tcp"
73+
cidr_blocks = [
74+
"0.0.0.0/0"]
75+
}
76+
77+
egress {
78+
protocol = -1
79+
from_port = 0
80+
to_port = 0
81+
cidr_blocks = [
82+
"0.0.0.0/0"]
83+
}
84+
}
85+
86+
resource "aws_instance" "worker" {
87+
count = "${var.cluster_size}"
88+
ami = "ami-965e6bf3"
89+
instance_type = "t2.small"
90+
key_name = "${aws_key_pair.mega-benchmarker.key_name}"
91+
subnet_id = "${aws_subnet.workers.id}"
92+
vpc_security_group_ids = [
93+
"${aws_security_group.workers.id}"]
94+
associate_public_ip_address = true
95+
}
96+
97+
output "public_ips" {
98+
value = "${join(", ", aws_instance.worker.*.public_ip)}"
99+
}
100+
output "private_ips" {
101+
value = "${join(", ", aws_instance.worker.*.private_ip)}"
102+
}

‎benchmark/azure/remote.tf‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
variable "access_key" {}
2+
variable "secret_key" {}
3+
variable "key_pair" {}
4+
variable "cluster_size" {
5+
default = 3
6+
}
7+
8+
variable "prefix" {
9+
default = "flamestream-benchmarks"
10+
}
11+
12+
provider "azurerm" {
13+
version = "=1.33.0"
14+
}
15+
16+
data "azurerm_subscription" "current" {}
17+
18+
resource "azurerm_resource_group" "main" {
19+
name = "${var.prefix}-resources"
20+
location = "West Europe"
21+
}
22+
23+
resource "azurerm_virtual_network" "main" {
24+
name = "${var.prefix}-network"
25+
address_space = ["10.0.0.0/16"]
26+
location = "${azurerm_resource_group.main.location}"
27+
resource_group_name = "${azurerm_resource_group.main.name}"
28+
}
29+
30+
resource "azurerm_subnet" "internal" {
31+
name = "${var.prefix}-internal"
32+
resource_group_name = "${azurerm_resource_group.main.name}"
33+
virtual_network_name = "${azurerm_virtual_network.main.name}"
34+
address_prefix = "10.0.2.0/24"
35+
}
36+
37+
resource "azurerm_public_ip" "manager" {
38+
name = "test-pip"
39+
location = "${azurerm_resource_group.main.location}"
40+
resource_group_name = "${azurerm_resource_group.main.name}"
41+
allocation_method = "Dynamic"
42+
idle_timeout_in_minutes = 30
43+
}
44+
45+
resource "azurerm_network_interface" "manager" {
46+
name = "${var.prefix}-manager"
47+
location = "${azurerm_resource_group.main.location}"
48+
resource_group_name = "${azurerm_resource_group.main.name}"
49+
50+
ip_configuration {
51+
name = "testconfiguration1"
52+
subnet_id = "${azurerm_subnet.internal.id}"
53+
private_ip_address_allocation = "Dynamic"
54+
public_ip_address_id = "${azurerm_public_ip.manager.id}"
55+
}
56+
}
57+
58+
resource "azurerm_managed_disk" "manager" {
59+
name = "${var.prefix}-manager"
60+
location = "${azurerm_resource_group.main.location}"
61+
resource_group_name = "${azurerm_resource_group.main.name}"
62+
storage_account_type = "Standard_LRS"
63+
create_option = "Copy"
64+
source_resource_id = "${data.azurerm_subscription.current.id}/resourceGroups/FlamestreamBenchmarks/providers/Microsoft.Compute/snapshots/FlamestreamBenchmarks"
65+
disk_size_gb = "30"
66+
}
67+
68+
resource "azurerm_virtual_machine" "manager" {
69+
name = "${var.prefix}-manager"
70+
location = "${azurerm_resource_group.main.location}"
71+
resource_group_name = "${azurerm_resource_group.main.name}"
72+
network_interface_ids = ["${azurerm_network_interface.manager.id}"]
73+
vm_size = "Standard_B1ms"
74+
75+
delete_os_disk_on_termination = true
76+
delete_data_disks_on_termination = true
77+
78+
storage_os_disk {
79+
name = "${azurerm_managed_disk.manager.name}"
80+
caching = "ReadWrite"
81+
create_option = "Attach"
82+
managed_disk_id = "${azurerm_managed_disk.manager.id}"
83+
os_type = "Linux"
84+
}
85+
os_profile_linux_config {
86+
disable_password_authentication = true
87+
ssh_keys {
88+
key_data = file("~/.ssh/id_rsa.pub")
89+
path = "/home/ubuntu/.ssh/authorized_keys"
90+
}
91+
}
92+
}
93+
94+
data "azurerm_public_ip" "manager" {
95+
name = "${azurerm_public_ip.manager.name}"
96+
resource_group_name = "${azurerm_virtual_machine.manager.resource_group_name}"
97+
}
98+
99+
resource "azurerm_network_interface" "workers" {
100+
count = var.cluster_size
101+
name = "${var.prefix}-worker-${count.index}"
102+
location = "${azurerm_resource_group.main.location}"
103+
resource_group_name = "${azurerm_resource_group.main.name}"
104+
105+
ip_configuration {
106+
name = "testconfiguration1"
107+
subnet_id = "${azurerm_subnet.internal.id}"
108+
private_ip_address_allocation = "Dynamic"
109+
}
110+
}
111+
112+
resource "azurerm_managed_disk" "workers" {
113+
count = var.cluster_size
114+
name = "${var.prefix}-worker-${count.index}"
115+
location = "${azurerm_resource_group.main.location}"
116+
resource_group_name = "${azurerm_resource_group.main.name}"
117+
storage_account_type = "Standard_LRS"
118+
create_option = "Copy"
119+
source_resource_id = "${data.azurerm_subscription.current.id}/resourceGroups/FlamestreamBenchmarks/providers/Microsoft.Compute/snapshots/FlamestreamBenchmarks"
120+
disk_size_gb = "30"
121+
}
122+
123+
resource "azurerm_virtual_machine" "workers" {
124+
count = var.cluster_size
125+
name = "${var.prefix}-worker-${count.index}"
126+
location = "${azurerm_resource_group.main.location}"
127+
resource_group_name = "${azurerm_resource_group.main.name}"
128+
network_interface_ids = ["${azurerm_network_interface.workers[count.index].id}"]
129+
vm_size = "Standard_B1ms"
130+
131+
delete_os_disk_on_termination = true
132+
delete_data_disks_on_termination = true
133+
134+
storage_os_disk {
135+
name = azurerm_managed_disk.workers[count.index].name
136+
caching = "ReadWrite"
137+
create_option = "Attach"
138+
managed_disk_id = azurerm_managed_disk.workers[count.index].id
139+
os_type = "Linux"
140+
}
141+
os_profile_linux_config {
142+
disable_password_authentication = true
143+
ssh_keys {
144+
key_data = file("~/.ssh/id_rsa.pub")
145+
path = "/home/ubuntu/.ssh/authorized_keys"
146+
}
147+
}
148+
}
149+
150+
output "manager_public_ip" {
151+
value = "${data.azurerm_public_ip.manager.ip_address}"
152+
}
153+
154+
output "manager_private_ip" {
155+
value = "${azurerm_network_interface.manager.private_ip_address}"
156+
}
157+
158+
output "worker_private_ips" {
159+
value = "${azurerm_network_interface.workers.*.private_ip_address}"
160+
}

0 commit comments

Comments
 (0)