Skip to content

Commit b4de460

Browse files
chaozwnchaozwn
committed
[feature-#1058] Add docker development docker-compose file, help quickly build dev environment.
1 parent 0d3c3fb commit b4de460

31 files changed

Lines changed: 1053 additions & 1 deletion

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,6 @@ chunjun-dist/
4747
*/.cache
4848
*/node_modules
4949
/website/public
50-
/website/public/page-data/
50+
/website/public/page-data/
51+
52+
chunjun-dev/data/**

‎chunjun-dev/.env‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# 设置时区
2+
TZ=Asia/Shanghai
3+
# 设置网络模式
4+
NETWORKS_DRIVER=bridge
5+
# 宿主机上代码存放的目录路径
6+
DATA_PATH_HOST=./data
7+
8+
# MYSQL57服务映射宿主机端口号，可在宿主机127.0.0.1:40057访问
9+
MYSQL57_PORT=40057
10+
MYSQL57_USERNAME=admin
11+
MYSQL57_PASSWORD=123456
12+
MYSQL57_ROOT_PASSWORD=123456
13+
14+
# MYSQL8服务映射宿主机端口号，可在宿主机127.0.0.1:40080访问
15+
MYSQL8_PORT=40080
16+
MYSQL8_USERNAME=admin
17+
MYSQL8_PASSWORD=123456
18+
MYSQL8_ROOT_PASSWORD=123456
19+
20+
# Redis5 服务映射宿主机端口号，可在宿主机127.0.0.1:46375访问
21+
REDIS5_PORT=46375
22+
REDIS5_PASSWORD=123456
23+
# Redis6 服务映射宿主机端口号，可在宿主机127.0.0.1:46376访问
24+
REDIS6_PORT=46376
25+
REDIS6_PASSWORD=123456
26+
27+
# Dm7 服务映射宿主机端口号，可在宿主机127.0.0.1:45237访问
28+
DM7_PORT=45237
29+
# Dm8 服务映射宿主机端口号，可在宿主机127.0.0.1:45238访问
30+
DM8_PORT=45238
31+
32+
# Zookeeper 服务映射宿主机端口号，可在宿主机127.0.0.1:42181访问
33+
ZOOKEEPER_PORT=42181
34+
35+
# kafka28 服务映射宿主机端口号，可在宿主机127.0.0.1:49092访问
36+
KAFKA28_PORT=49092
37+
38+
# kudu 服务映射宿主机端口号，可在宿主机127.0.0.1:49092访问
39+
KUDU_MASTER_PORT=7051
40+
KUDU_TSERVER_PORT=7050

‎chunjun-dev/README.md‎

Whitespace-only changes.

‎chunjun-dev/README_CH.md‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# Chunjun开发环境构建
2+
一个基于 `docker` 的 `chunjun` 本地开发运行环境，也可以用于快速搭建chunjun的local开发环境。
3+
4+
5+
## 使用
6+
### 1. 按需修改 .env 配置 (主要包含了一些端口，如果被占用，请自行更改)
7+
~~~
8+
# 设置时区
9+
TZ=Asia/Shanghai
10+
# 设置网络模式
11+
NETWORKS_DRIVER=bridge
12+
# 宿主机上代码存放的目录路径
13+
DATA_PATH_HOST=./data
14+
15+
# MYSQL57服务映射宿主机端口号，可在宿主机127.0.0.1:40057访问
16+
MYSQL57_PORT=40057
17+
MYSQL57_USERNAME=admin
18+
MYSQL57_PASSWORD=123456
19+
MYSQL57_ROOT_PASSWORD=123456
20+
21+
# MYSQL8服务映射宿主机端口号，可在宿主机127.0.0.1:40080访问
22+
MYSQL8_PORT=40080
23+
MYSQL8_USERNAME=admin
24+
MYSQL8_PASSWORD=123456
25+
MYSQL8_ROOT_PASSWORD=123456
26+
27+
# Redis5 服务映射宿主机端口号，可在宿主机127.0.0.1:46375访问
28+
REDIS5_PORT=46375
29+
REDIS5_PASSWORD=123456
30+
# Redis6 服务映射宿主机端口号，可在宿主机127.0.0.1:46376访问
31+
REDIS6_PORT=46376
32+
REDIS6_PASSWORD=123456
33+
34+
# Dm7 服务映射宿主机端口号，可在宿主机127.0.0.1:45237访问
35+
DM7_PORT=45237
36+
# Dm8 服务映射宿主机端口号，可在宿主机127.0.0.1:45238访问
37+
DM8_PORT=45238
38+
39+
# Zookeeper 服务映射宿主机端口号，可在宿主机127.0.0.1:42181访问
40+
ZOOKEEPER_PORT=42181
41+
42+
# kafka28 服务映射宿主机端口号，可在宿主机127.0.0.1:49092访问
43+
KAFKA28_PORT=49092
44+
45+
# kudu 服务映射宿主机端口号，可在宿主机127.0.0.1:49092访问
46+
KUDU_MASTER_PORT=7051
47+
KUDU_TSERVER_PORT=7050
48+
~~~
49+
50+
### 2.启动服务
51+
- 启动全部服务
52+
```bash
53+
docker-compose up -d
54+
```
55+
- 按需启动部分服务
56+
```bash
57+
# 其他的服务名，请查看docker-compose.yml
58+
docker-compose up -d chunjun-redis5 chunjun-mysql8
59+
```
60+
61+
### 3.停止服务
62+
- 停止全部服务
63+
```bash
64+
docker-compose stop
65+
```
66+
- 按需停止部分服务
67+
```bash
68+
# 其他的服务名，请查看docker-compose.yml
69+
docker-compose stop chunjun-redis5 chunjun-mysql8
70+
```
71+
72+
### 4.开发示例
73+
开发方式一般有两种，一种是使用IDEA(或者Jetbrain的gateway)和vscode进行开发，核心思路是远程到我们的app镜像当中开发。 为什么要远程到app镜像当中开发，主要是为了解决一些host映射的问题，我们在容器中debug代码，就不需要把host映射到主机，从而解决hbase，hive的一些host映射问题。
74+
#### 4.1 启动开发环境
75+
```shell
76+
cd chunjun-dev
77+
# 等待所有的镜像构建完毕
78+
docker-compose up -d
79+
# 构建部分镜像，(推荐)
80+
docker-compose up chunjun-app
81+
```
82+
![](images/docker_containers.png)
83+
84+
#### 4.2 进入app container，克隆代码
85+
```shell
86+
docker exec -it chunjun-dev_chunjun-app_1 bash
87+
88+
cd /root/app/code
89+
90+
git clone https://github.com/DTStack/chunjun.git
91+
92+
cd chunjun
93+
94+
mvn clean package -DskipTests
95+
```
96+
![](images/build_success.png)
97+
98+
#### 4.3 IDE打开，这边使用的是Gateway
99+
```shell
100+
账号密码 root/root，等待进入IDE
101+
```
102+
![](images/new_connection.png)
103+
![](images/chunjun_dev.png)
104+
![](images/gateway_ide.png)
105+
106+
#### 4.4 第一个测试
107+
```shell
108+
// 在chunjun的代码仓库下创建一个job是目录，创建文件mysql_to_print.json，复制下面的json
109+
运行类 com.dtstack.chunjun.client.Launcher
110+
运行参数 -job /root/app/code/chunjun/jobs/mysql_to_print.json -jobType sync -chunjunDistDir /root/app/code/chunjun/chunjun-dist
111+
```
112+
```json
113+
{
114+
"job": {
115+
"content": [
116+
{
117+
"reader": {
118+
"name": "mysqlreader",
119+
"parameter": {
120+
"column": [
121+
{
122+
"name": "Host",
123+
"type": "string"
124+
},
125+
{
126+
"name": "User",
127+
"type": "string"
128+
}
129+
],
130+
"username": "root",
131+
"password": "123456",
132+
"connection": [
133+
{
134+
"jdbcUrl": [
135+
"jdbc:mysql://chunjun-mysql5.7:3306/mysql?useSSL=false"
136+
],
137+
"table": [
138+
"user"
139+
]
140+
}
141+
]
142+
}
143+
},
144+
"writer": {
145+
"name": "streamwriter",
146+
"parameter": {
147+
"print": true
148+
}
149+
}
150+
}
151+
],
152+
"setting": {
153+
"speed": {
154+
"channel": 1,
155+
"bytes": 0
156+
}
157+
}
158+
}
159+
}
160+
```
161+
![](images/success.png)
162+
```shell
163+
//运行截图
164+
1.如果你能得到相同的结果，那么开发环境就构建成功了.
165+
2.请注意保护chunjun-dev/data/app目录下的文件,你改动的代码都会被同步到这个文件夹
166+
```
167+
### 5.kafka操作(ps：kafka已开启自动创建topic的功能)
168+
```bash
169+
docker exec -it chunjun-dev_chunjun-kafka2.8_1 /bin/sh
170+
cd /opt/kafka/bin/
171+
172+
//创建topic
173+
./kafka-topics.sh --create --zookeeper chunjun-zookeeper:2181 --replication-factor 1 -partitions 1 --topic chunjun-test
174+
175+
//查看所有topic
176+
./kafka-topics.sh --list --zookeeper chunjun-zookeeper:2181
177+
178+
//测试producer，启动后输入hello,world
179+
./kafka-console-producer.sh --bootstrap-server chunjun-kafka2.8:49092 --topic chunjun-test
180+
181+
//测试consumer,看到hello-world，表示安装完毕
182+
./kafka-console-consumer.sh --bootstrap-server chunjun-kafka2.8:49092 --topic chunjun-test --from-beginning
183+
184+
```

‎chunjun-dev/app/Dockerfile‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
FROM archlinux:base-devel
2+
3+
WORKDIR /tmp
4+
ENV SHELL /bin/bash
5+
ADD mirrorlist /etc/pacman.d/mirrorlist
6+
RUN yes | pacman -Syu
7+
RUN yes | pacman -S git zsh
8+
VOLUME [ "/root/", "/root/.local/share/chezmoi", "/root/.vscode-server/extensions", "/var/lib/docker" ]
9+
# end
10+
11+
# bash
12+
ADD bashrc /root/.bashrc
13+
# end
14+
15+
# basic tools
16+
RUN yes | pacman -S curl tree wget vim vi
17+
ENV EDITOR=vim
18+
ENV VISUAL=vim
19+
# end
20+
21+
# tools
22+
RUN yes | pacman -S fzf openssh exa the_silver_searcher fd rsync \
23+
&& ssh-keygen -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key &&\
24+
ssh-keygen -t dsa -N '' -f /etc/ssh/ssh_host_dsa_key
25+
26+
RUN echo "root:root" | chpasswd
27+
28+
# Java
29+
RUN yes | pacman -S jdk8-openjdk
30+
ENV JAVA_HOME=/usr/lib/jvm/default
31+
ENV PATH=$JAVA_HOME/bin:$PATH
32+
33+
# maven
34+
ENV MAVEN_VERSION 3.8.6
35+
ENV MAVEN_HOME=/usr/lib/mvn
36+
ENV PATH=$MAVEN_HOME/bin:$PATH
37+
38+
RUN wget http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
39+
tar -zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz && \
40+
rm apache-maven-$MAVEN_VERSION-bin.tar.gz && \
41+
mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
42+
43+
COPY ./settings.xml /usr/lib/mvn/conf
44+
RUN mkdir -p /root/.m2
45+
RUN mkdir -p /root/.m2/repository
46+
COPY ./settings.xml /root/.m2/
47+
COPY ./clean_maven.sh /root
48+
49+
RUN mkdir -p /root/app/code
50+
WORKDIR /root/app/code
51+
52+
COPY ./sshd_config /etc/ssh
53+
54+
EXPOSE 22
55+
56+
CMD ["/usr/sbin/sshd", "-D"]

‎chunjun-dev/app/bashrc‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
[ `alias | grep "^ls=" | wc -l` != 0 ] && unalias ls
2+
alias gst='git status -sb'
3+
alias ga='git add'
4+
alias gc='git commit'
5+
alias gl='git pull'
6+
alias gp='git push'
7+
alias glog="git log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit -- | less"
8+
alias ls='exa'
9+
alias ll='ls -lh'
10+
alias la='ls -alh'
11+
alias fq='trojan -c ~/.config/trojan.conf > /tmp/trojan.log 2>&1 &'
12+
alias pc='proxychains -f ~/.config/proxychains.conf'
13+
14+
[ -f ~/.bash_aliases.local ] && source ~/.bash_aliases.local
15+
[ -f ~/.config/z/z.sh ] && source ~/.config/z/z.sh
16+
[ -f /usr/local/rvm/scripts/rvm ] && source /usr/local/rvm/scripts/rvm
17+
export FZF_DEFAULT_COMMAND='ag -g ""'
18+
export TZ='Asia/Shanghai'
19+
export EDITOR=vim
20+
export VISUAL=vim
21+
export PYTHONUNBUFFERED=1
22+
export PATH="/root/.local/bin:$PATH"
23+
export JAVA_HOME=/usr/lib/jvm/default
24+
export PATH=$JAVA_HOME/bin:$PATH
25+
export MAVEN_HOME=/usr/lib/mvn
26+
export PATH=$MAVEN_HOME/bin:$PATH
27+
28+
f() {
29+
local file
30+
q=$1
31+
32+
file=$(ag -l -g ""| fzf --query="$q" --select-1 --exit-0 -x)
33+
if [ -n "$file" ] ;then
34+
code "$file"
35+
fi
36+
}
37+
fd() {
38+
local file
39+
local dir
40+
file=$(fzf +m -q "$1") && dir=$(dirname "$file") && cd "$dir"
41+
}
42+
fs(){
43+
local file
44+
q=$1
45+
if [ -z "$q"] ;then
46+
q="."
47+
fi
48+
result=$(ag "$q" | fzf)
49+
IFS=':' read file line other <<< "$result"
50+
[ -n "$file" ] && code -g "$file":"$line";
51+
}
52+
53+
[ `alias | grep "^z=" | wc -l` != 0 ] && unalias z
54+
j() {
55+
if [[ -z "$*" ]]; then
56+
cd "$(_z -l 2>&1 | fzf +s | sed 's/^[0-9,.]* *//')"
57+
else
58+
_last_z_args="$@"
59+
_z "$@"
60+
fi
61+
}
62+
jj() {
63+
cd "$(_z -l 2>&1 | sed 's/^[0-9,.]* *//' | fzf -q $_last_z_args)"
64+
}

‎chunjun-dev/app/clean_maven.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/sh
2+
#!/bin/bash
3+
4+
REPO_PATH="/root/.m2/repository"
5+
6+
echo "正在清理"$REPO_PATH"目录下的.lastUpdated后缀文件"
7+
8+
find $REPO_PATH -name "*.lastUpdated" | xargs rm -fr
9+
10+
echo "完成清理"

0 commit comments

Comments
 (0)